Merge remote-tracking branch 'remotes/origin/develop' into fix-BIOPET-640

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.core.report
 import java.io._
 
 import nl.lumc.sasc.biopet.core.ToolCommandFunction
-import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
+import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample, Run }
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ LibraryId, SampleId }
 import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
@@ -40,7 +40,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
   /** Report builder object */
   def builder: ReportBuilder
 
-  def toolObject = builder
+  def toolObject: ReportBuilder = builder
 
   @Input(required = true)
   var summaryDbFile: File = _
@@ -114,20 +114,24 @@ trait ReportBuilder extends ToolCommand {
   private var setSummary: SummaryDb = _
 
   /** Retrival of summary, read only */
-  final def summary = setSummary
+  final def summary: SummaryDb = setSummary
 
   private var setRunId: Int = 0
 
-  final def runId = setRunId
+  final def runId: Int = setRunId
+
+  private var _setRun: Run = _
+
+  final def run: Run = _setRun
 
   private var _setPipelines = Seq[Pipeline]()
-  final def pipelines = _setPipelines
+  final def pipelines: Seq[Pipeline] = _setPipelines
   private var _setModules = Seq[Module]()
-  final def modules = _setModules
+  final def modules: Seq[Module] = _setModules
   private var _setSamples = Seq[Sample]()
-  final def samples = _setSamples
+  final def samples: Seq[Sample] = _setSamples
   private var _setLibraries = Seq[Library]()
-  final def libraries = _setLibraries
+  final def libraries: Seq[Library] = _setLibraries
 
   /** default args that are passed to all page withing the report */
   def pageArgs: Map[String, Any] = Map()
@@ -136,13 +140,13 @@ trait ReportBuilder extends ToolCommand {
   private var total = 0
 
   private var _sampleId: Option[Int] = None
-  protected[report] def sampleId = _sampleId
+  protected[report] def sampleId: Option[Int] = _sampleId
   private var _libId: Option[Int] = None
-  protected[report] def libId = _libId
+  protected[report] def libId: Option[Int] = _libId
 
   case class ExtFile(resourcePath: String, targetPath: String)
 
-  def extFiles = List(
+  def extFiles: List[ExtFile] = List(
     "css/bootstrap_dashboard.css",
     "css/bootstrap.min.css",
     "css/bootstrap-theme.min.css",
@@ -183,6 +187,7 @@ trait ReportBuilder extends ToolCommand {
       case _ =>
     }
 
+    _setRun = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
     _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
     _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
     _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
@@ -264,6 +269,7 @@ trait ReportBuilder extends ToolCommand {
       val rootPath = "./" + Array.fill(path.size)("../").mkString
       args ++ page.args ++
         Map("page" -> page,
+          "run" -> run,
           "path" -> path,
           "outputDir" -> pageOutputDir,
           "rootPath" -> rootPath,

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/BqsrGather.scala

+package nl.lumc.sasc.biopet.extensions.gatk
+
+import java.io.File
+import java.util
+
+import org.broadinstitute.gatk.engine.recalibration.BQSRGatherer
+import org.broadinstitute.gatk.queue.function.InProcessFunction
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by pjvanthof on 05/04/2017.
+ */
+class BqsrGather extends InProcessFunction {
+
+  @Input(required = true)
+  var inputBqsrFiles: List[File] = _
+
+  @Output(required = true)
+  var outputBqsrFile: File = _
+
+  def run(): Unit = {
+    val l = new util.ArrayList[File]()
+    inputBqsrFiles.foreach(l.add(_))
+    val gather = new BQSRGatherer
+    gather.gather(l, outputBqsrFile)
+  }
+}

flexiprep/src/main/resources/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepFastQcPlot.ssp

+#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Run)
 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
@@ -10,6 +11,7 @@
 #import(nl.lumc.sasc.biopet.core.report.ReportBuilder.ec)
 <%@ var summary: SummaryDb %>
 <%@ var runId: Int %>
+<%@ var run: Run %>
 <%@ var sampleId: Option[Int] %>
 <%@ var libId: Option[Int] %>
 <%@ var plot: String %>

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/Mapping.scala

@@ -37,7 +37,9 @@ import org.broadinstitute.gatk.queue.QScript
 
 import scala.math._
 
-// TODO: documentation
+/**
+ * This pipeline doing a alignment to a given reference genome
+ */
 class Mapping(val parent: Configurable) extends QScript with SummaryQScript with SampleLibraryTag with Reference {
 
   def this() = this(null)
@@ -79,7 +81,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
   protected var platform: String = config("platform", default = "illumina")
 
   /** Readgroup platform unit */
-  protected var platformUnit: String = config("platform_unit", default = "na")
+  protected var platformUnit: Option[String] = config("platform_unit")
 
   /** Readgroup sequencing center */
   protected var readgroupSequencingCenter: Option[String] = config("readgroup_sequencing_center")
@@ -97,9 +99,9 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
 
   protected var paired: Boolean = false
   val flexiprep = new Flexiprep(this)
-  def finalBamFile: File = if (skipMarkduplicates) {
-    new File(outputDir, outputName + ".bam")
-  } else new File(outputDir, outputName + ".dedup.bam")
+  def mergedBamFile = new File(outputDir, outputName + ".bam")
+  def finalBamFile: File = if (skipMarkduplicates) mergedBamFile
+  else new File(outputDir, outputName + ".dedup.bam")
 
   override def defaults: Map[String, Any] = Map(
     "gsnap" -> Map("batch" -> 4),
@@ -258,21 +260,25 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     }
 
     var bamFile = bamFiles.head
+
+    if (!chunking) require(bamFile == mergedBamFile)
+    else {
+      val mergeSamFile = MergeSamFiles(this, bamFiles, mergedBamFile)
+      mergeSamFile.isIntermediate = !keepFinalBamFile || !skipMarkduplicates
+      add(mergeSamFile)
+      bamFile = mergeSamFile.output
+    }
+
     if (!skipMarkduplicates) {
       bamFile = new File(outputDir, outputName + ".dedup.bam")
-      val md = MarkDuplicates(this, bamFiles, finalBamFile)
+      val md = MarkDuplicates(this, mergedBamFile :: Nil, finalBamFile)
       md.isIntermediate = !keepFinalBamFile
       add(md)
       addSummarizable(md, "mark_duplicates")
-    } else if (skipMarkduplicates && chunking) {
-      val mergeSamFile = MergeSamFiles(this, bamFiles, finalBamFile)
-      mergeSamFile.isIntermediate = !keepFinalBamFile
-      add(mergeSamFile)
-      bamFile = mergeSamFile.output
     }
 
     if (!skipMetrics) {
-      val bamMetrics = BamMetrics(this, bamFile, new File(outputDir, "metrics"), sampleId, libId)
+      val bamMetrics = BamMetrics(this, finalBamFile, new File(outputDir, "metrics"), sampleId, libId)
       addAll(bamMetrics.functions)
       addSummaryQScript(bamMetrics)
     }
@@ -379,7 +385,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     hisat2.R2 = R2
     hisat2.rgId = Some(readgroupId)
     hisat2.rg +:= s"PL:$platform"
-    hisat2.rg +:= s"PU:$platformUnit"
+    platformUnit.foreach(x => hisat2.rg +:= s"PU:$x")
     libId match {
       case Some(id)  => hisat2.rg +:= s"LB:$id"
       case otherwise => ;
@@ -452,7 +458,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     RG += "SM:" + sampleId.get + ","
     RG += "LB:" + libId.get + ","
     if (readgroupDescription != null) RG += "DS" + readgroupDescription + ","
-    RG += "PU:" + platformUnit + ","
+    platformUnit.foreach(x => RG += "PU:" + x + ",")
     if (predictedInsertsize.getOrElse(0) > 0) RG += "PI:" + predictedInsertsize.get + ","
     if (readgroupSequencingCenter.isDefined) RG += "CN:" + readgroupSequencingCenter.get + ","
     if (readgroupDate != null) RG += "DT:" + readgroupDate + ","
@@ -497,7 +503,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     bowtie2.rgId = Some(readgroupId)
     bowtie2.rg +:= ("LB:" + libId.get)
     bowtie2.rg +:= ("PL:" + platform)
-    bowtie2.rg +:= ("PU:" + platformUnit)
+    platformUnit.foreach(x => bowtie2.rg +:= ("PU:" + x))
     bowtie2.rg +:= ("SM:" + sampleId.get)
     bowtie2.R1 = R1
     bowtie2.R2 = R2
@@ -554,7 +560,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     addOrReplaceReadGroups.RGID = readgroupId
     addOrReplaceReadGroups.RGLB = libId.get
     addOrReplaceReadGroups.RGPL = platform
-    addOrReplaceReadGroups.RGPU = platformUnit
+    addOrReplaceReadGroups.RGPU = platformUnit.getOrElse(readgroupId)
     addOrReplaceReadGroups.RGSM = sampleId.get
     if (readgroupSequencingCenter.isDefined) addOrReplaceReadGroups.RGCN = readgroupSequencingCenter.get
     if (readgroupDescription.isDefined) addOrReplaceReadGroups.RGDS = readgroupDescription.get
@@ -568,7 +574,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
     var RG: String = "@RG\\t" + "ID:" + readgroupId + "\\t"
     RG += "LB:" + libId.get + "\\t"
     RG += "PL:" + platform + "\\t"
-    RG += "PU:" + platformUnit + "\\t"
+    platformUnit.foreach(x => RG += "PU:" + x + "\\t")
     RG += "SM:" + sampleId.get + "\\t"
     if (readgroupSequencingCenter.isDefined) RG += "CN:" + readgroupSequencingCenter.get + "\\t"
     if (readgroupDescription.isDefined) RG += "DS:" + readgroupDescription.get + "\\t"

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/Shiva.scala

@@ -50,8 +50,12 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     "unifiedgenotyper" -> Map("stand_call_conf" -> 30, "stand_emit_conf" -> 0)
   )
 
+  lazy val usePrintReads: Boolean = config("use_printreads", default = true)
+
   /** Method to make the variantcalling namespace of shiva */
-  def makeVariantcalling(multisample: Boolean, sample: Option[String] = None, library: Option[String] = None): ShivaVariantcalling with QScript = {
+  def makeVariantcalling(multisample: Boolean,
+                         sample: Option[String] = None,
+                         library: Option[String] = None): ShivaVariantcalling with QScript = {
     if (multisample) new ShivaVariantcalling(qscript) {
       override def namePrefix = "multisample"
       override def configNamespace: String = "shivavariantcalling"
@@ -79,7 +83,10 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     /** Class to generate jobs for a library */
     class Library(libId: String) extends super.Library(libId) {
 
-      override def summaryFiles = super.summaryFiles ++ variantcalling.map("final" -> _.finalFile)
+      override def summaryFiles = super.summaryFiles ++
+        variantcalling.map("final" -> _.finalFile) ++
+        bqsrFile.map("baserecal" -> _) ++
+        bqsrAfterFile.map("baserecal_after" -> _)
 
       lazy val useIndelRealigner: Boolean = config("use_indel_realigner", default = true)
       lazy val useBaseRecalibration: Boolean = {
@@ -89,20 +96,28 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
           logger.warn("No Known site found, skipping base recalibration, file: " + inputBam)
         c && br.knownSites.nonEmpty
       }
+      lazy val useAnalyzeCovariates: Boolean = if (useBaseRecalibration) config("use_analyze_covariates", default = true) else false
+
+      lazy val bqsrFile: Option[File] = if (useBaseRecalibration) Some(createFile("baserecal")) else None
+      lazy val bqsrAfterFile: Option[File] = if (useAnalyzeCovariates) Some(createFile("baserecal.after")) else None
 
       override def keepFinalBamfile = super.keepFinalBamfile && !useIndelRealigner && !useBaseRecalibration
 
-      override def preProcessBam = if (useIndelRealigner && useBaseRecalibration)
+      override def bamFile = mapping.map(_.mergedBamFile)
+
+      override def preProcessBam = if (useIndelRealigner && usePrintReads)
         bamFile.map(swapExt(libDir, _, ".bam", ".realign.baserecal.bam"))
       else if (useIndelRealigner) bamFile.map(swapExt(libDir, _, ".bam", ".realign.bam"))
-      else if (useBaseRecalibration) bamFile.map(swapExt(libDir, _, ".bam", ".baserecal.bam"))
+      else if (usePrintReads) bamFile.map(swapExt(libDir, _, ".bam", ".baserecal.bam"))
       else bamFile
 
       /** Library specific settings */
-      override def summarySettings = Map(
+      override def summarySettings = super.summarySettings ++ Map(
         "library_variantcalling" -> variantcalling.isDefined,
         "use_indel_realigner" -> useIndelRealigner,
-        "use_base_recalibration" -> useBaseRecalibration)
+        "use_base_recalibration" -> useBaseRecalibration,
+        "useAnalyze_covariates" -> useAnalyzeCovariates
+      )
 
       lazy val variantcalling = if (config("library_variantcalling", default = false).asBoolean &&
         (bamFile.isDefined || preProcessBam.isDefined)) {
@@ -115,11 +130,11 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
 
         if (useIndelRealigner && useBaseRecalibration) {
           val file = addIndelRealign(bamFile.get, libDir, isIntermediate = true)
-          addBaseRecalibrator(file, libDir, libraries.size > 1)
+          addBaseRecalibrator(file, libDir, libraries.size > 1, usePrintReads)
         } else if (useIndelRealigner) {
           addIndelRealign(bamFile.get, libDir, libraries.size > 1)
         } else if (useBaseRecalibration) {
-          addBaseRecalibrator(bamFile.get, libDir, libraries.size > 1)
+          addBaseRecalibrator(bamFile.get, libDir, libraries.size > 1, usePrintReads)
         }
 
         variantcalling.foreach(vc => {
@@ -131,6 +146,41 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
           add(vc)
         })
       }
+
+      /** Adds base recalibration jobs */
+      def addBaseRecalibrator(inputBam: File, dir: File, isIntermediate: Boolean, usePrintreads: Boolean): File = {
+        require(bqsrFile.isDefined, "bqsrFile should contain something at this point")
+        val baseRecalibrator = BaseRecalibrator(qscript, inputBam, bqsrFile.get) // at this point bqsrFile should exist
+
+        if (baseRecalibrator.knownSites.isEmpty) return inputBam
+        add(baseRecalibrator)
+
+        if (useAnalyzeCovariates) {
+          val baseRecalibratorAfter = BaseRecalibrator(qscript, inputBam, bqsrAfterFile.get)
+          baseRecalibratorAfter.BQSR = bqsrFile
+          add(baseRecalibratorAfter)
+          add(AnalyzeCovariates(qscript, baseRecalibrator.out, baseRecalibratorAfter.out, swapExt(dir, inputBam, ".bam", ".baserecal.pdf")))
+        }
+        if (usePrintreads) {
+          val printReads = PrintReads(qscript, inputBam, swapExt(dir, inputBam, ".bam", ".baserecal.bam"))
+          printReads.BQSR = Some(baseRecalibrator.out)
+          printReads.isIntermediate = isIntermediate
+          add(printReads)
+          printReads.out
+        } else inputBam
+      }
+
+    } // end of library
+
+    lazy val bqsrFile: Option[File] = {
+      val files = libraries.flatMap(_._2.bqsrFile).toList
+      if (files.isEmpty) None else {
+        val gather = new BqsrGather
+        gather.inputBqsrFiles = files
+        gather.outputBqsrFile = createFile("baserecal")
+        add(gather)
+        Some(gather.outputBqsrFile)
+      }
     }
 
     lazy val variantcalling = if (config("single_sample_variantcalling", default = false).asBoolean) {
@@ -164,7 +214,7 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
         })
       }
     }
-  }
+  } // End of sample
 
   lazy val multisampleVariantCalling = if (config("multisample_variantcalling", default = true).asBoolean) {
     Some(makeVariantcalling(multisample = true))
@@ -192,7 +242,17 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     multisampleVariantCalling.foreach(vc => {
       vc.outputDir = new File(outputDir, "variantcalling")
       vc.inputBams = samples.flatMap { case (sampleId, sample) => sample.preProcessBam.map(sampleId -> _) }
+      if (!usePrintReads)
+        vc.inputBqsrFiles = samples.flatMap { case (sampleId, sample) => sample.bqsrFile.map(sampleId -> _) }
       add(vc)
+      if (!usePrintReads) {
+        import variantcallers._
+        if (vc.callers.exists(_ match {
+          case _: HaplotypeCaller | _: HaplotypeCallerAllele | _: HaplotypeCallerGvcf => false
+          case _: UnifiedGenotyper | _: UnifiedGenotyperAllele => false
+          case _ => true
+        })) logger.warn("Not all variantcallers chosen can read BQSR files, All non-GATK")
+      }
 
       annotation.foreach { toucan =>
         toucan.outputDir = new File(outputDir, "annotation")
@@ -221,7 +281,8 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     "sv_calling" -> svCalling.isDefined,
     "cnv_calling" -> cnvCalling.isDefined,
     "regions_of_interest" -> roiBedFiles.map(_.getName.stripSuffix(".bed")),
-    "amplicon_bed" -> ampliconBedFile.map(_.getName.stripSuffix(".bed"))
+    "amplicon_bed" -> ampliconBedFile.map(_.getName.stripSuffix(".bed")),
+    "use_print_reads" -> usePrintReads
   )
 
   /** Adds indel realignment jobs */
@@ -237,28 +298,6 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
     indelRealigner.out
   }
 
-  /** Adds base recalibration jobs */
-  def addBaseRecalibrator(inputBam: File, dir: File, isIntermediate: Boolean): File = {
-    val baseRecalibrator = BaseRecalibrator(this, inputBam, swapExt(dir, inputBam, ".bam", ".baserecal"))
-
-    if (baseRecalibrator.knownSites.isEmpty) return inputBam
-    add(baseRecalibrator)
-
-    if (config("use_analyze_covariates", default = true).asBoolean) {
-      val baseRecalibratorAfter = BaseRecalibrator(this, inputBam, swapExt(dir, inputBam, ".bam", ".baserecal.after"))
-      baseRecalibratorAfter.BQSR = Some(baseRecalibrator.out)
-      add(baseRecalibratorAfter)
-
-      add(AnalyzeCovariates(this, baseRecalibrator.out, baseRecalibratorAfter.out, swapExt(dir, inputBam, ".bam", ".baserecal.pdf")))
-    }
-
-    val printReads = PrintReads(this, inputBam, swapExt(dir, inputBam, ".bam", ".baserecal.bam"))
-    printReads.BQSR = Some(baseRecalibrator.out)
-    printReads.isIntermediate = isIntermediate
-    add(printReads)
-
-    printReads.out
-  }
 }
 
 /** This object give a default main method to the pipelines */

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaVariantcalling.scala

@@ -46,6 +46,8 @@ class ShivaVariantcalling(val parent: Configurable) extends QScript