Merge remote-tracking branch 'remotes/origin/develop' into fix-shiva_fails

public/bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetrics.scala

@@ -19,7 +19,7 @@ import java.io.File
 
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import nl.lumc.sasc.biopet.core.summary.SummaryQScript
-import nl.lumc.sasc.biopet.core.{ PipelineCommand, SampleLibraryTag }
+import nl.lumc.sasc.biopet.core.{ BiopetFifoPipe, PipelineCommand, SampleLibraryTag }
 import nl.lumc.sasc.biopet.extensions.bedtools.{ BedtoolsCoverage, BedtoolsIntersect }
 import nl.lumc.sasc.biopet.extensions.picard._
 import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsFlagstat
@@ -151,33 +151,24 @@ class BamMetrics(val root: Configurable) extends QScript with SummaryQScript wit
       val targetDir = new File(outputDir, targetName)
 
       val biStrict = BedtoolsIntersect(this, inputBam, intervals.bed,
-        output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.strict.bam"),
+        output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.strict.sam"),
         minOverlap = config("strict_intersect_overlap", default = 1.0))
-      biStrict.isIntermediate = true
-      add(biStrict)
-      add(SamtoolsFlagstat(this, biStrict.output, targetDir))
       val biopetFlagstatStrict = BiopetFlagstat(this, biStrict.output, targetDir)
-      add(biopetFlagstatStrict)
       addSummarizable(biopetFlagstatStrict, targetName + "_biopet_flagstat_strict")
+      add(new BiopetFifoPipe(this, List(biStrict, biopetFlagstatStrict)))
 
       val biLoose = BedtoolsIntersect(this, inputBam, intervals.bed,
-        output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.loose.bam"),
+        output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.loose.sam"),
         minOverlap = config("loose_intersect_overlap", default = 0.01))
-      biLoose.isIntermediate = true
-      add(biLoose)
-      add(SamtoolsFlagstat(this, biLoose.output, targetDir))
       val biopetFlagstatLoose = BiopetFlagstat(this, biLoose.output, targetDir)
-      add(biopetFlagstatLoose)
       addSummarizable(biopetFlagstatLoose, targetName + "_biopet_flagstat_loose")
+      add(new BiopetFifoPipe(this, List(biLoose, biopetFlagstatLoose)))
 
-      val coverageFile = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".coverage")
-
-      //FIXME:should use piping
-      add(BedtoolsCoverage(this, inputBam, intervals.bed, coverageFile, depth = true), isIntermediate = true)
-      val covStats = CoverageStats(this, coverageFile, targetDir)
-      covStats.title = Some("Coverage for " + targetName)
-      covStats.subTitle = Some(".")
-      add(covStats)
+      val bedCov = BedtoolsCoverage(this, intervals.bed, inputBam, depth = true)
+      val covStats = CoverageStats(this, targetDir, inputBam.getName.stripSuffix(".bam") + ".coverage")
+      covStats.title = Some("Coverage Plot")
+      covStats.subTitle = Some(s"for file '$targetName.bed'")
+      add(bedCov | covStats)
       addSummarizable(covStats, targetName + "_cov_stats")
     }
 

public/bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/scripts/CoverageStats.scala

@@ -26,7 +26,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
 class CoverageStats(val root: Configurable) extends PythonCommandLineFunction with Summarizable {
   setPythonScript("bedtools_cov_stats.py")
 
-  @Input(doc = "Input file")
+  @Input(doc = "Input file", required = false)
   var input: File = _
 
   @Output(doc = "output File")
@@ -41,7 +41,7 @@ class CoverageStats(val root: Configurable) extends PythonCommandLineFunction wi
   override def defaultCoreMemory = 9.0
 
   def cmdLine = getPythonCommand +
-    required(input) +
+    (if (inputAsStdin) " - " else required(input)) +
     required("--plot", plot) +
     optional("--title", title) +
     optional("--subtitle", subTitle) +
@@ -55,11 +55,10 @@ class CoverageStats(val root: Configurable) extends PythonCommandLineFunction wi
 }
 
 object CoverageStats {
-  def apply(root: Configurable, input: File, outputDir: File): CoverageStats = {
+  def apply(root: Configurable, outputDir: File, name: String): CoverageStats = {
     val coverageStats = new CoverageStats(root)
-    coverageStats.input = input
-    coverageStats.output = new File(outputDir, input.getName + ".stats")
-    coverageStats.plot = new File(outputDir, input.getName + ".stats.png")
+    coverageStats.output = new File(outputDir, name + ".stats")
+    coverageStats.plot = new File(outputDir, name + ".stats.png")
     coverageStats
   }
 }

public/bammetrics/src/test/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetricsTest.scala

@@ -81,12 +81,7 @@ class BamMetricsTest extends TestNGSuite with Matchers {
     bammetrics.functions.count(_.isInstanceOf[CollectMultipleMetrics]) shouldBe 1
     bammetrics.functions.count(_.isInstanceOf[CalculateHsMetrics]) shouldBe (if (amplicon) 1 else 0)
     bammetrics.functions.count(_.isInstanceOf[CollectTargetedPcrMetrics]) shouldBe (if (amplicon) 1 else 0)
-    bammetrics.functions.count(_.isInstanceOf[BiopetFlagstat]) shouldBe (1 + (regions * 2))
-    bammetrics.functions.count(_.isInstanceOf[SamtoolsFlagstat]) shouldBe (1 + (regions * 2))
-    bammetrics.functions.count(_.isInstanceOf[BedtoolsIntersect]) shouldBe (regions * 2)
-
-    bammetrics.functions.count(_.isInstanceOf[BedtoolsCoverage]) shouldBe regions
-    bammetrics.functions.count(_.isInstanceOf[CoverageStats]) shouldBe regions
+    bammetrics.functions.count(_.isInstanceOf[BiopetFlagstat]) shouldBe 1
   }
 
   // remove temporary run directory all tests in the class have been run

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/bedtools/BedtoolsCoverage.scala

@@ -41,32 +41,26 @@ class BedtoolsCoverage(val root: Configurable) extends Bedtools {
   @Argument(doc = "diffStrand", required = false)
   var diffStrand: Boolean = false
 
-  var inputTag = "-a"
-
-  override def beforeCmd() {
-    if (input.getName.endsWith(".bam")) inputTag = "-abam"
-  }
-
   override def defaultCoreMemory = 4.0
 
   /** Returns command to execute */
   def cmdLine = required(executable) + required("coverage") +
-    required(inputTag, input) +
+    required("-a", input) +
     required("-b", intersectFile) +
     conditional(depth, "-d") +
     conditional(sameStrand, "-s") +
     conditional(diffStrand, "-S") +
-    " > " + required(output)
+    (if (outputAsStsout) "" else " > " + required(output))
 }
 
 object BedtoolsCoverage {
   /** Returns defaul bedtools coverage */
-  def apply(root: Configurable, input: File, intersect: File, output: File,
-            depth: Boolean = true, sameStrand: Boolean = false, diffStrand: Boolean = false): BedtoolsCoverage = {
+  def apply(root: Configurable, input: File, intersect: File, output: Option[File] = None,
+            depth: Boolean = false, sameStrand: Boolean = false, diffStrand: Boolean = false): BedtoolsCoverage = {
     val bedtoolsCoverage = new BedtoolsCoverage(root)
     bedtoolsCoverage.input = input
     bedtoolsCoverage.intersectFile = intersect
-    bedtoolsCoverage.output = output
+    output.foreach(bedtoolsCoverage.output = _)
     bedtoolsCoverage.depth = depth
     bedtoolsCoverage.sameStrand = sameStrand
     bedtoolsCoverage.diffStrand = diffStrand

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/bedtools/BedtoolsIntersect.scala

@@ -40,6 +40,8 @@ class BedtoolsIntersect(val root: Configurable) extends Bedtools {
 
   var inputTag = "-a"
 
+  var ubam = false
+
   override def beforeCmd() {
     if (input.getName.endsWith(".bam")) inputTag = "-abam"
   }
@@ -50,6 +52,7 @@ class BedtoolsIntersect(val root: Configurable) extends Bedtools {
     required("-b", intersectFile) +
     optional("-f", minOverlap) +
     conditional(count, "-c") +
+    conditional(ubam, "-ubam") +
     " > " + required(output)
 }
 
@@ -61,6 +64,7 @@ object BedtoolsIntersect {
     bedtoolsIntersect.input = input
     bedtoolsIntersect.intersectFile = intersect
     bedtoolsIntersect.output = output
+    if (output.getName.endsWith(".sam")) bedtoolsIntersect.ubam = true
     bedtoolsIntersect.minOverlap = minOverlap
     bedtoolsIntersect.count = count
     bedtoolsIntersect

public/sage/src/main/scala/nl/lumc/sasc/biopet/pipelines/sage/Sage.scala

@@ -173,19 +173,22 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
   }
 
   def addBedtoolsCounts(bamFile: File, outputPrefix: String, outputDir: File) {
-    val bedtoolsSense = BedtoolsCoverage(this, bamFile, squishedCountBed, new File(outputDir, outputPrefix + ".genome.sense.coverage"),
+    val bedtoolsSense = BedtoolsCoverage(this, bamFile, squishedCountBed,
+      output = Some(new File(outputDir, outputPrefix + ".genome.sense.coverage")),
       depth = false, sameStrand = true, diffStrand = false)
     val countSense = new BedtoolsCoverageToCounts(this)
     countSense.input = bedtoolsSense.output
     countSense.output = new File(outputDir, outputPrefix + ".genome.sense.counts")
 
-    val bedtoolsAntisense = BedtoolsCoverage(this, bamFile, squishedCountBed, new File(outputDir, outputPrefix + ".genome.antisense.coverage"),
+    val bedtoolsAntisense = BedtoolsCoverage(this, bamFile, squishedCountBed,
+      output = Some(new File(outputDir, outputPrefix + ".genome.antisense.coverage")),
       depth = false, sameStrand = false, diffStrand = true)
     val countAntisense = new BedtoolsCoverageToCounts(this)
     countAntisense.input = bedtoolsAntisense.output
     countAntisense.output = new File(outputDir, outputPrefix + ".genome.antisense.counts")
 
-    val bedtools = BedtoolsCoverage(this, bamFile, squishedCountBed, new File(outputDir, outputPrefix + ".genome.coverage"),
+    val bedtools = BedtoolsCoverage(this, bamFile, squishedCountBed,
+      output = Some(new File(outputDir, outputPrefix + ".genome.coverage")),
       depth = false, sameStrand = false, diffStrand = false)
     val count = new BedtoolsCoverageToCounts(this)
     count.input = bedtools.output