Commit da34ed23 authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub

Merge branch 'develop' into fix-BIOPET-627

parents 1c4740c8 6f3ac600
......@@ -67,6 +67,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
val skipWriteDependencies: Boolean = config("skip_write_dependencies", default = false)
val writeHtmlReport: Boolean = config("write_html_report", default = true)
/** Script from queue itself, final to force some checks for each pipeline and write report */
final def script() {
outputDir = config("output_dir")
......@@ -126,8 +128,6 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
}
})
val writeHtmlReport: Boolean = config("write_html_report", default = true)
if (writeHtmlReport) {
logger.info("Adding report")
this match {
......
......@@ -22,7 +22,16 @@ import org.broadinstitute.gatk.queue.function.CommandLineFunction
*/
trait CommandLineResources extends CommandLineFunction with Configurable {
/**
* This value is overruling threads method when this is set.
* This can be used to limit the number of threads on a global level.
*/
lazy val maxThreads: Option[Int] = config("max_threads")
/** To set an other default threads this method need to be override */
def defaultThreads = 1
/** This method will get and set the cores requested */
final def threads = nCoresRequest match {
case Some(i) => i
case _ =>
......@@ -72,7 +81,6 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
* @return number of threads
*/
private def getThreads(default: Int): Int = {
val maxThreads: Option[Int] = config("maxthreads")
val threads: Int = config("threads", default = default)
maxThreads match {
case Some(max) => if (max > threads) threads else max
......@@ -120,11 +128,12 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
protected def combineResources(commands: List[CommandLineResources]): Unit = {
commands.foreach(_.setResources())
nCoresRequest = Some(commands.map(_.threads).sum + threadsCorrection)
nCoresRequest = nCoresRequest.map(x => if (x > maxThreads.getOrElse(x)) maxThreads.getOrElse(x) else x)
_coreMemory = commands.map(cmd => cmd.coreMemory * (cmd.threads.toDouble / threads.toDouble)).sum
memoryLimit = Some(_coreMemory * threads)
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) threads else 1))
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) threads else 1)) + "G")
memoryLimit = Some(_coreMemory * nCoresRequest.getOrElse(threads))
residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) nCoresRequest.getOrElse(threads) else 1))
vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) nCoresRequest.getOrElse(threads) else 1)) + "G")
}
}
......@@ -135,6 +135,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
}
md5sum.input = file
md5sum.output = new File(file.getParentFile, file.getName + ".md5")
md5sum.jobOutputFile = new File(file.getParentFile, s".${file.getName}.md5.md5sum.out")
// Need to not write a md5 file outside the outputDir
if (!file.getAbsolutePath.startsWith(outputDir.getAbsolutePath))
......
......@@ -64,7 +64,7 @@ class CommandLineResourcesTest extends TestNGSuite with Matchers {
@Test
def testMaxThreads(): Unit = {
val cmd = new CommandLineFunctionMock(Map("maxthreads" -> 5, "threads" -> 10)) with CommandLineResources
val cmd = new CommandLineFunctionMock(Map("max_threads" -> 5, "threads" -> 10)) with CommandLineResources
cmd.threads shouldBe 5
}
......
......@@ -84,7 +84,6 @@ class FreeC(val parent: Configurable) extends BiopetCommandLineFunction with Ref
var minExpectedGC: Option[Double] = config("minExpectedGC")
var maxExpectedGC: Option[Double] = config("maxExpectedGC")
var minimalSubclonePresence: Option[Int] = config("minimalSubclonePresence")
var maxThreads: Int = getThreads
var noisyData: Boolean = config("noisyData", default = false)
//var outputDir: File
......
......@@ -36,7 +36,8 @@ class VcfStatsForSv(val parent: Configurable) extends ToolCommandFunction with S
@Output(required = true)
var outputFile: File = _
override def defaultCoreMemory = 1.0
override def defaultResidentFactor = 2.0
override def defaultVmemFactor = 3.0
override def cmdLine = super.cmdLine +
required("-i", inputFile) +
......
......@@ -38,8 +38,7 @@ object VcfStatsForSv extends ToolCommand {
val stats: Map[String, Any] = getVariantCounts(cmdArgs.inputFile, cmdArgs.histBinBoundaries)
ConfigUtils.mapToYamlFile(stats, cmdArgs.outputFile)
ConfigUtils.mapToJsonFile(stats, cmdArgs.outputFile)
}
/** Parses a vcf-file and counts sv-s by type and size. Sv-s are divided to different size classes, the parameter histogramBinBoundaries gives the boundaries between these classes. */
......
......@@ -168,6 +168,12 @@ object ConfigUtils extends Logging {
writer.close()
}
def mapToJsonFile(map: Map[String, Any], outputFile: File) = {
val writer = new PrintWriter(outputFile)
writer.println(anyToJson(map).toString())
writer.close()
}
/** Convert json to native scala map/values */
def jsonToMap(json: Json): Map[String, Any] = {
var output: Map[String, Any] = Map()
......
......@@ -113,7 +113,7 @@ case class BedRecordList(val chrRecords: Map[String, List[BedRecord]], val heade
}
/** This return the fraction of the regions comparing to a length */
def fractionOf(length: Long): Double = length.toDouble / length
def fractionOf(length: Long): Double = this.length.toDouble / length.toDouble
/** This return the fraction of the regions comparing to a reference */
def fractionOfReference(dict: SAMSequenceDictionary): Double = fractionOf(dict.getReferenceLength)
......
......@@ -41,6 +41,8 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
/** Make a final fastq files, by default only when flexiprep is the main pipeline */
var keepQcFastqFiles: Boolean = config("keepQcFastqFiles", default = parent == null)
override def defaults = super.defaults ++ Map("max_threads" -> 4)
/** Returns files to store in summary */
def summaryFiles: Map[String, File] = {
Map("input_R1" -> inputR1, "output_R1" -> fastqR1Qc) ++
......@@ -226,6 +228,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
override def summaryDeps: List[File] = qcCmdR1.summaryDeps ::: qcCmdR2.summaryDeps ::: super.summaryDeps
}
pipe.jobOutputFile = new File(outDir, ".qc_cmd.out")
pipe.deps ::= fastqcR1.output
pipe.deps ::= fastqcR2.output
pipe.deps ::= R1_in
......@@ -241,6 +244,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
R2 = Some(fqSync.outputFastq2)
} else {
qcCmdR1.nCoresRequest = Some(2)
qcCmdR1.jobOutputFile = new File(outDir, ".qc_cmd.out")
add(qcCmdR1)
R1 = qcCmdR1.output
}
......
......@@ -86,6 +86,9 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
/** Readgroup sequencing center */
protected var readgroupSequencingCenter: Option[String] = config("readgroup_sequencing_center")
/** Readgroup library */
protected var readgroupLibrary: Option[String] = config("readgroup_library")
/** Readgroup description */
protected var readgroupDescription: Option[String] = config("readgroup_description")
......@@ -572,7 +575,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
/** Returns readgroup for bwa */
def getReadGroupBwa: String = {
var RG: String = "@RG\\t" + "ID:" + readgroupId + "\\t"
RG += "LB:" + libId.get + "\\t"
readgroupLibrary.foreach(lb => RG += "LB:" + lb + "\\t")
RG += "PL:" + platform + "\\t"
platformUnit.foreach(x => RG += "PU:" + x + "\\t")
RG += "SM:" + sampleId.get + "\\t"
......
......@@ -44,12 +44,6 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
Some(shiva)
}
override def defaults = Map(
"haplotypecaller" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0),
"genotypegvcfs" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0),
"unifiedgenotyper" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0)
)
lazy val usePrintReads: Boolean = config("use_printreads", default = true)
/** Method to make the variantcalling namespace of shiva */
......
......@@ -20,9 +20,14 @@ import nl.lumc.sasc.biopet.extensions.Pysvtools
import nl.lumc.sasc.biopet.extensions.tools.VcfStatsForSv
import nl.lumc.sasc.biopet.pipelines.shiva.svcallers._
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import nl.lumc.sasc.biopet.utils.{ BamUtils, Logging }
import org.broadinstitute.gatk.queue.QScript
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/**
* Common trait for ShivaVariantcalling
*
......@@ -43,7 +48,16 @@ class ShivaSvCalling(val parent: Configurable) extends QScript with SummaryQScri
/** Executed before script */
def init(): Unit = {
if (inputBamsArg.nonEmpty) inputBams = BamUtils.sampleBamMap(inputBamsArg)
if (inputBamsArg.nonEmpty) {
inputBams = BamUtils.sampleBamMap(inputBamsArg)
val db = SummaryDb.openSqliteSummary(summaryDbFile)
for (sampleName <- inputBams.keys) {
if (Await.result(db.getSampleId(summaryRunId, sampleName), Duration.Inf).isEmpty) {
db.createSample(sampleName, summaryRunId)
}
}
}
outputMergedVCF = new File(outputDir, "allsamples.merged.vcf")
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment