Commit 4cac2929 authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub
Browse files

Merge branch 'develop' into fix-BIOPET-604

parents 2dca0473 7810cb76
......@@ -26,7 +26,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
......
......@@ -24,7 +24,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
......
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
......@@ -72,14 +73,14 @@
case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList
}
}#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
#for (libId <- libs)
#{ val libName = libId.map(l => allLibraries.find(_.id == l).get.name) }#
#if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{
val statsPaths = fields.map(x => x -> List("metrics", x.toUpperCase)).toMap
val results = summary.getStatKeys(runId, "bammetrics", "wgs", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
val results = summary.getStatKeys(runId, "bammetrics", "rna", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}#
#for (field <- fields)
<td>${results(field)}</td>
......
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
......
......@@ -306,12 +306,12 @@ object BammetricsReport extends ReportBuilder {
sampleId: Option[Int] = None,
libraryId: Option[Int] = None): Unit = {
val statsPaths = Map(
"position" -> List("rna", "histogram", "normalized_position"),
"count" -> List("rna", "histogram", "All_Reads.normalized_coverage")
"position" -> List("histogram", "normalized_position"),
"count" -> List("histogram", "All_Reads.normalized_coverage")
)
writePlotFromSummary(outputDir, prefix, summary, libraryLevel, sampleId, libraryId, statsPaths,
"coverage", "count", "bammetrics", "rna",
"position", "count", "bammetrics", "rna",
"Relative position", "Coverage", "Rna coverage")
}
......
......@@ -31,7 +31,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
......
......@@ -101,8 +101,8 @@ class BastyTest extends TestNGSuite with Matchers {
pipeline.functions.count(_.isInstanceOf[MarkDuplicates]) shouldBe (numberLibs + numberSamples)
// Gatk preprocess
pipeline.functions.count(_.isInstanceOf[IndelRealigner]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
pipeline.functions.count(_.isInstanceOf[RealignerTargetCreator]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
pipeline.functions.count(_.isInstanceOf[IndelRealigner]) shouldBe (if (realign) numberSamples else 0)
pipeline.functions.count(_.isInstanceOf[RealignerTargetCreator]) shouldBe (if (realign) numberSamples else 0)
pipeline.functions.count(_.isInstanceOf[BaseRecalibrator]) shouldBe (if (dbsnp && baseRecalibration) (numberLibs * 2) else 0)
pipeline.functions.count(_.isInstanceOf[PrintReads]) shouldBe (if (dbsnp && baseRecalibration) numberLibs else 0)
......
......@@ -21,7 +21,7 @@
<parent>
<artifactId>Biopet</artifactId>
<groupId>nl.lumc.sasc</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -67,6 +67,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
val skipWriteDependencies: Boolean = config("skip_write_dependencies", default = false)
val writeHtmlReport: Boolean = config("write_html_report", default = true)
/** Script from queue itself, final to force some checks for each pipeline and write report */
final def script() {
outputDir = config("output_dir")
......@@ -126,8 +128,6 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
}
})
val writeHtmlReport: Boolean = config("write_html_report", default = true)
if (writeHtmlReport) {
logger.info("Adding report")
this match {
......
......@@ -22,7 +22,16 @@ import org.broadinstitute.gatk.queue.function.CommandLineFunction
*/
trait CommandLineResources extends CommandLineFunction with Configurable {
/**
* This value is overruling threads method when this is set.
* This can be used to limit the number of threads on a global level.
*/
lazy val maxThreads: Option[Int] = config("max_threads")
/** To set an other default threads this method need to be override */
def defaultThreads = 1
/** This method will get and set the cores requested */
final def threads = nCoresRequest match {
case Some(i) => i
case _ =>
......@@ -72,7 +81,6 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
* @return number of threads
*/
private def getThreads(default: Int): Int = {
val maxThreads: Option[Int] = config("maxthreads")
val threads: Int = config("threads", default = default)
maxThreads match {
case Some(max) => if (max > threads) threads else max
......@@ -120,11 +128,12 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
protected def combineResources(commands: List[CommandLineResources]): Unit = {
commands.foreach(_.setResources())
nCoresRequest = Some(commands.map(_.threads).sum + threadsCorrection)
nCoresRequest = nCoresRequest.map(x => if (x > maxThreads.getOrElse(x)) maxThreads.getOrElse(x) else x)
_coreMemory = commands.map(cmd => cmd.coreMemory * (cmd.threads.toDouble / threads.toDouble)).sum
memoryLimit = Some(_coreMemory * threads)
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) threads else 1))
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) threads else 1)) + "G")
memoryLimit = Some(_coreMemory * nCoresRequest.getOrElse(threads))
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) nCoresRequest.getOrElse(threads) else 1))
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) nCoresRequest.getOrElse(threads) else 1)) + "G")
}
}
......@@ -134,11 +134,13 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
required("echo") + required("error_on_capture " + input.toString) + " > " + required(output)
}
md5sum.input = file
md5sum.output = new File(file.getParentFile, file.getName + ".md5")
// Need to not write a md5 file outside the outputDir
if (!file.getAbsolutePath.startsWith(outputDir.getAbsolutePath))
md5sum.output = new File(outputDir, ".md5" + file.getAbsolutePath + ".md5")
md5sum.output = if (file.getAbsolutePath.startsWith(outputDir.getAbsolutePath))
new File(file.getParentFile, file.getName + ".md5")
else {
// Need to not write a md5 file outside the outputDir
new File(outputDir, ".md5" + file.getAbsolutePath + ".md5")
}
md5sum.jobOutputFile = new File(md5sum.output.getParentFile, s".${file.getName}.md5.md5sum.out")
writeSummary.deps :+= md5sum.output
SummaryQScript.md5sumCache += file -> md5sum.output
......
......@@ -64,7 +64,7 @@ class CommandLineResourcesTest extends TestNGSuite with Matchers {
@Test
def testMaxThreads(): Unit = {
val cmd = new CommandLineFunctionMock(Map("maxthreads" -> 5, "threads" -> 10)) with CommandLineResources
val cmd = new CommandLineFunctionMock(Map("max_threads" -> 5, "threads" -> 10)) with CommandLineResources
cmd.threads shouldBe 5
}
......
......@@ -21,7 +21,7 @@
<parent>
<artifactId>Biopet</artifactId>
<groupId>nl.lumc.sasc</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -84,7 +84,6 @@ class FreeC(val parent: Configurable) extends BiopetCommandLineFunction with Ref
var minExpectedGC: Option[Double] = config("minExpectedGC")
var maxExpectedGC: Option[Double] = config("maxExpectedGC")
var minimalSubclonePresence: Option[Int] = config("minimalSubclonePresence")
var maxThreads: Int = getThreads
var noisyData: Boolean = config("noisyData", default = false)
//var outputDir: File
......
/**
* Biopet is built on top of GATK Queue for building bioinformatic
* pipelines. It is mainly intended to support LUMC SHARK cluster which is running
* SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
* should also be able to execute Biopet tools and pipelines.
*
* Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
*
* Contact us at: sasc@lumc.nl
*
* A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
* license; For commercial users or users who do not want to follow the AGPL
* license, please contact us to obtain a separate license.
*/
package nl.lumc.sasc.biopet.extensions.gatk
import java.io.File
......
......@@ -24,7 +24,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
......
......@@ -21,7 +21,7 @@
<parent>
<artifactId>Biopet</artifactId>
<groupId>nl.lumc.sasc</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -36,7 +36,8 @@ class VcfStatsForSv(val parent: Configurable) extends ToolCommandFunction with S
@Output(required = true)
var outputFile: File = _
override def defaultCoreMemory = 1.0
override def defaultResidentFactor = 2.0
override def defaultVmemFactor = 3.0
override def cmdLine = super.cmdLine +
required("-i", inputFile) +
......
......@@ -21,7 +21,7 @@
<parent>
<artifactId>Biopet</artifactId>
<groupId>nl.lumc.sasc</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -21,7 +21,7 @@
<parent>
<artifactId>Biopet</artifactId>
<groupId>nl.lumc.sasc</groupId>
<version>0.9.0-SNAPSHOT</version>
<version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment