Merge pull request #88 from biopet/fix-BIOPET-645

Corrections on threads

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -67,6 +67,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
 
   val skipWriteDependencies: Boolean = config("skip_write_dependencies", default = false)
 
+  val writeHtmlReport: Boolean = config("write_html_report", default = true)
+
   /** Script from queue itself, final to force some checks for each pipeline and write report */
   final def script() {
     outputDir = config("output_dir")
@@ -126,8 +128,6 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
       }
     })
 
-    val writeHtmlReport: Boolean = config("write_html_report", default = true)
-
     if (writeHtmlReport) {
       logger.info("Adding report")
       this match {

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/CommandLineResources.scala

@@ -22,7 +22,16 @@ import org.broadinstitute.gatk.queue.function.CommandLineFunction
  */
 trait CommandLineResources extends CommandLineFunction with Configurable {
 
+  /**
+   * This value is overruling threads method when this is set.
+   * This can be used to limit the number of threads on a global level.
+   */
+  lazy val maxThreads: Option[Int] = config("max_threads")
+
+  /** To set an other default threads this method need to be override */
   def defaultThreads = 1
+
+  /** This method will get and set the cores requested */
   final def threads = nCoresRequest match {
     case Some(i) => i
     case _ =>
@@ -72,7 +81,6 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
    * @return number of threads
    */
   private def getThreads(default: Int): Int = {
-    val maxThreads: Option[Int] = config("maxthreads")
     val threads: Int = config("threads", default = default)
     maxThreads match {
       case Some(max) => if (max > threads) threads else max
@@ -120,11 +128,12 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
   protected def combineResources(commands: List[CommandLineResources]): Unit = {
     commands.foreach(_.setResources())
     nCoresRequest = Some(commands.map(_.threads).sum + threadsCorrection)
+    nCoresRequest = nCoresRequest.map(x => if (x > maxThreads.getOrElse(x)) maxThreads.getOrElse(x) else x)
 
     _coreMemory = commands.map(cmd => cmd.coreMemory * (cmd.threads.toDouble / threads.toDouble)).sum
-    memoryLimit = Some(_coreMemory * threads)
-    residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) threads else 1))
-    vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) threads else 1)) + "G")
+    memoryLimit = Some(_coreMemory * nCoresRequest.getOrElse(threads))
+    residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor * (if (multiplyRssThreads) nCoresRequest.getOrElse(threads) else 1))
+    vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry)) * (if (multiplyVmemThreads) nCoresRequest.getOrElse(threads) else 1)) + "G")
   }
 
 }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala

@@ -135,6 +135,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
           }
           md5sum.input = file
           md5sum.output = new File(file.getParentFile, file.getName + ".md5")
+          md5sum.jobOutputFile = new File(file.getParentFile, s".${file.getName}.md5.md5sum.out")
 
           // Need to not write a md5 file outside the outputDir
           if (!file.getAbsolutePath.startsWith(outputDir.getAbsolutePath))

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/CommandLineResourcesTest.scala

@@ -64,7 +64,7 @@ class CommandLineResourcesTest extends TestNGSuite with Matchers {
 
   @Test
   def testMaxThreads(): Unit = {
-    val cmd = new CommandLineFunctionMock(Map("maxthreads" -> 5, "threads" -> 10)) with CommandLineResources
+    val cmd = new CommandLineFunctionMock(Map("max_threads" -> 5, "threads" -> 10)) with CommandLineResources
 
     cmd.threads shouldBe 5
   }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/freec/FreeC.scala

@@ -84,7 +84,6 @@ class FreeC(val parent: Configurable) extends BiopetCommandLineFunction with Ref
   var minExpectedGC: Option[Double] = config("minExpectedGC")
   var maxExpectedGC: Option[Double] = config("maxExpectedGC")
   var minimalSubclonePresence: Option[Int] = config("minimalSubclonePresence")
-  var maxThreads: Int = getThreads
 
   var noisyData: Boolean = config("noisyData", default = false)
   //var outputDir: File

biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/VcfStatsForSv.scala

@@ -36,7 +36,8 @@ class VcfStatsForSv(val parent: Configurable) extends ToolCommandFunction with S
   @Output(required = true)
   var outputFile: File = _
 
-  override def defaultCoreMemory = 1.0
+  override def defaultResidentFactor = 2.0
+  override def defaultVmemFactor = 3.0
 
   override def cmdLine = super.cmdLine +
     required("-i", inputFile) +

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/vcfstats/VcfStatsForSv.scala

@@ -38,8 +38,7 @@ object VcfStatsForSv extends ToolCommand {
 
     val stats: Map[String, Any] = getVariantCounts(cmdArgs.inputFile, cmdArgs.histBinBoundaries)
 
-    ConfigUtils.mapToYamlFile(stats, cmdArgs.outputFile)
-
+    ConfigUtils.mapToJsonFile(stats, cmdArgs.outputFile)
   }
 
   /** Parses a vcf-file and counts sv-s by type and size. Sv-s are divided to different size classes, the parameter histogramBinBoundaries gives the boundaries between these classes. */

biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/ConfigUtils.scala

@@ -168,6 +168,12 @@ object ConfigUtils extends Logging {
     writer.close()
   }
 
+  def mapToJsonFile(map: Map[String, Any], outputFile: File) = {
+    val writer = new PrintWriter(outputFile)
+    writer.println(anyToJson(map).toString())
+    writer.close()
+  }
+
   /** Convert json to native scala map/values */
   def jsonToMap(json: Json): Map[String, Any] = {
     var output: Map[String, Any] = Map()

biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/intervals/BedRecordList.scala

@@ -113,7 +113,7 @@ case class BedRecordList(val chrRecords: Map[String, List[BedRecord]], val heade
   }
 
   /** This return the fraction of the regions comparing to a length */
-  def fractionOf(length: Long): Double = length.toDouble / length
+  def fractionOf(length: Long): Double = this.length.toDouble / length.toDouble
 
   /** This return the fraction of the regions comparing to a reference */
   def fractionOfReference(dict: SAMSequenceDictionary): Double = fractionOf(dict.getReferenceLength)

flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala

@@ -41,6 +41,8 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
   /** Make a final fastq files, by default only when flexiprep is the main pipeline */
   var keepQcFastqFiles: Boolean = config("keepQcFastqFiles", default = parent == null)
 
+  override def defaults = super.defaults ++ Map("max_threads" -> 4)
+
   /** Returns files to store in summary */
   def summaryFiles: Map[String, File] = {
     Map("input_R1" -> inputR1, "output_R1" -> fastqR1Qc) ++
@@ -226,6 +228,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
         override def summaryDeps: List[File] = qcCmdR1.summaryDeps ::: qcCmdR2.summaryDeps ::: super.summaryDeps
       }
 
+      pipe.jobOutputFile = new File(outDir, ".qc_cmd.out")
       pipe.deps ::= fastqcR1.output
       pipe.deps ::= fastqcR2.output
       pipe.deps ::= R1_in
@@ -241,6 +244,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
       R2 = Some(fqSync.outputFastq2)
     } else {
       qcCmdR1.nCoresRequest = Some(2)
+      qcCmdR1.jobOutputFile = new File(outDir, ".qc_cmd.out")
       add(qcCmdR1)
       R1 = qcCmdR1.output
     }

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/Mapping.scala

@@ -86,6 +86,9 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
   /** Readgroup sequencing center */
   protected var readgroupSequencingCenter: Option[String] = config("readgroup_sequencing_center")
 
+  /** Readgroup library */
+  protected var readgroupLibrary: Option[String] = config("readgroup_library")
+
   /** Readgroup description */
   protected var readgroupDescription: Option[String] = config("readgroup_description")
 
@@ -572,7 +575,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
   /** Returns readgroup for bwa */
   def getReadGroupBwa: String = {
     var RG: String = "@RG\\t" + "ID:" + readgroupId + "\\t"
-    RG += "LB:" + libId.get + "\\t"
+    readgroupLibrary.foreach(lb => RG += "LB:" + lb + "\\t")
     RG += "PL:" + platform + "\\t"
     platformUnit.foreach(x => RG += "PU:" + x + "\\t")
     RG += "SM:" + sampleId.get + "\\t"

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/Shiva.scala

@@ -44,12 +44,6 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     Some(shiva)
   }
 
-  override def defaults = Map(
-    "haplotypecaller" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0),
-    "genotypegvcfs" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0),
-    "unifiedgenotyper" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0)
-  )
-
   lazy val usePrintReads: Boolean = config("use_printreads", default = true)
 
   /** Method to make the variantcalling namespace of shiva */

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaSvCalling.scala

@@ -20,9 +20,14 @@ import nl.lumc.sasc.biopet.extensions.Pysvtools
 import nl.lumc.sasc.biopet.extensions.tools.VcfStatsForSv
 import nl.lumc.sasc.biopet.pipelines.shiva.svcallers._
 import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import nl.lumc.sasc.biopet.utils.{ BamUtils, Logging }
 import org.broadinstitute.gatk.queue.QScript
 
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+import scala.concurrent.ExecutionContext.Implicits.global
+
 /**
  * Common trait for ShivaVariantcalling
  *
@@ -43,7 +48,16 @@ class ShivaSvCalling(val parent: Configurable) extends QScript with SummaryQScri
 
   /** Executed before script */
   def init(): Unit = {
-    if (inputBamsArg.nonEmpty) inputBams = BamUtils.sampleBamMap(inputBamsArg)
+    if (inputBamsArg.nonEmpty) {
+      inputBams = BamUtils.sampleBamMap(inputBamsArg)
+
+      val db = SummaryDb.openSqliteSummary(summaryDbFile)
+      for (sampleName <- inputBams.keys) {
+        if (Await.result(db.getSampleId(summaryRunId, sampleName), Duration.Inf).isEmpty) {
+          db.createSample(sampleName, summaryRunId)
+        }
+      }
+    }
     outputMergedVCF = new File(outputDir, "allsamples.merged.vcf")
   }