Commit 8e661da6 authored by Sander Bollen's avatar Sander Bollen

Merge branch 'develop' into fix-BIOPET-513

parents b22079f4 8d4df927
[submodule "gatk"]
path = gatk
url = https://github.com/biopet/gatk.git
node('local') {
try {
stage('Init') {
tool 'JDK 8u102'
tool 'Apache Maven 3.3.9'
}
stage('Checkout') {
checkout scm
sh 'git submodule update --init --recursive'
}
stage('Build and Test') {
withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
}
}
stage('Report tests') {
junit '*/target/surefire-reports/*.xml'
}
if(currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
currentBuild.result = "SUCCESS"
slackSend (color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else {
slackSend (color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
}
} catch (e) {
if(currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
currentBuild.result = "FAILED"
}
slackSend (color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
throw e
}
}
......@@ -345,8 +345,7 @@ object BammetricsReport extends ReportBuilder {
val pathValues: Map[String, Array[Any]] = paths.map {
case (key, path) =>
val value = summary.getValueAsArray(sampleId, libId, path: _*)
require(value.isDefined, s"Sample: $sampleId, library: $libId on path: '${path.mkString(",")}' does not exist in summary")
key -> value.get
key -> value.getOrElse(Array())
}
require(pathValues.map(_._2.size).toList.distinct.size == 1, s"Arrays in summary does not have the same number of values, $paths")
pathValues
......
......@@ -47,6 +47,11 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
private def changeScript(file: File): Unit = {
val lines = Source.fromFile(file).getLines().toList
val writer = new PrintWriter(file)
remoteCommand match {
case "bash" => writer.println("#!/bin/bash")
case "sh" => writer.println("#!/bin/sh")
case _ => writer.println(s"#!$remoteCommand")
}
writer.println("set -eubf")
writer.println("set -o pipefail")
lines.foreach(writer.println)
......@@ -66,8 +71,9 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
changeScript(new File(jt.getArgs.head.toString))
jt.setRemoteCommand(remoteCommand)
case ps: ProcessSettings =>
changeScript(new File(ps.getCommand.tail.head))
ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail)
changeScript(new File(ps.getCommand.last))
if (ps.getCommand.head != "srun")
ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail)
}
/**
......
......@@ -48,14 +48,19 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
def coreMemory = _coreMemory
/** This value is for SGE and is defined in seconds */
protected val maxWalltimeLimit: Option[Int] = config("max_walltime_limit")
wallTime = config("max_walltime")
/** This value is specific for slurm */
qualityOfSerice = config("quality_of_serice")
var retry = 0
override def freezeFieldValues(): Unit = {
setResources()
if (useSge && vmem.isDefined) jobResourceRequests :+= s"h_vmem=${vmem.get}"
if (useSge && maxWalltimeLimit.isDefined) jobResourceRequests :+= s"h_rt=${maxWalltimeLimit.get}"
if (useSge) {
vmem.foreach(v => jobResourceRequests :+= s"h_vmem=$v")
wallTime.foreach(t => jobResourceRequests :+= s"h_rt=$t")
}
super.freezeFieldValues()
}
......@@ -106,6 +111,7 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
if (vmem.isDefined) jobResourceRequests = jobResourceRequests.filterNot(_.contains("h_vmem="))
if (retry > 0) logger.info("Auto raise memory on retry")
retry += 1
waitBeforeJob = waitBeforeJob.map(_ + (retry * 10))
this.freezeFieldValues()
}
......@@ -117,8 +123,8 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
_coreMemory = commands.map(cmd => cmd.coreMemory * (cmd.threads.toDouble / threads.toDouble)).sum
memoryLimit = Some(_coreMemory * threads)
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor)
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry))) + "G")
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) threads else 1))
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) threads else 1)) + "G")
}
}
Subproject commit 7805065f00df24c58948ea9961a4e9393ae64c70
......@@ -24,6 +24,7 @@
<version>0.8.0-SNAPSHOT</version>
<modules>
<module>gatk</module>
<module>biopet-package</module>
<module>bam2wig</module>
<module>bammetrics</module>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment