Merge remote-tracking branch 'origin/develop' into feature-scoverage

Conflicts:
	protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala

@@ -87,6 +87,12 @@ class ShivaTest extends TestNGSuite with Matchers {
 
 object ShivaTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)
@@ -130,8 +136,8 @@ object ShivaTest {
   val sample1 = Map(
     "samples" -> Map("sample1" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "1_1_R1.fq",
-        "R2" -> "1_1_R2.fq"
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
       )
     )
     )))
@@ -139,12 +145,12 @@ object ShivaTest {
   val sample2 = Map(
     "samples" -> Map("sample3" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "3_1_R1.fq",
-        "R2" -> "3_1_R2.fq"
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
       ),
       "lib2" -> Map(
-        "R1" -> "3_2_R1.fq",
-        "R2" -> "3_2_R2.fq"
+        "R1" -> inputTouch("2_2_R1.fq"),
+        "R2" -> inputTouch("2_2_R2.fq")
       )
     )
     )))

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala

@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
     val map = Map("variantcallers" -> callers.toList)
     val pipeline = initPipeline(map)
 
-    pipeline.inputBams = (for (n <- 1 to bams) yield new File("bam_" + n + ".bam")).toList
+    pipeline.inputBams = (for (n <- 1 to bams) yield ShivaVariantcallingTest.inputTouch("bam_" + n + ".bam")).toList
 
     val illegalArgumentException = pipeline.inputBams.isEmpty ||
       (!raw && !bcftools &&
@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
 
 object ShivaVariantcallingTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): File = {
+    val file = new File(outputDir, "input" + File.separator + name).getAbsoluteFile
+    Files.touch(file)
+    file
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/bam2wig/src/main/scala/nl/lumc/sasc/biopet/pipelines/bamtobigwig/Bam2Wig.scala

@@ -35,6 +35,7 @@ class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript {
   var bamFile: File = null
 
   def init(): Unit = {
+    inputFiles :+= new InputFile(bamFile)
   }
 
   def biopetScript(): Unit = {

public/bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetrics.scala

@@ -71,7 +71,8 @@ class BamMetrics(val root: Configurable) extends QScript with SummaryQScript wit
   }
 
   /** executed before script */
-  def init() {
+  def init(): Unit = {
+    inputFiles :+= new InputFile(inputBam)
   }
 
   /** Script to add jobs */

public/bammetrics/src/test/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetricsTest.scala

@@ -69,7 +69,7 @@ class BamMetricsTest extends TestNGSuite with Matchers {
       Map("regions_of_interest" -> (1 to rois).map("roi_" + _ + ".bed").toList)
     val bammetrics: BamMetrics = initPipeline(map)
 
-    bammetrics.inputBam = new File("input.bam")
+    bammetrics.inputBam = BamMetricsTest.bam
     bammetrics.sampleId = Some("1")
     bammetrics.libId = Some("1")
     bammetrics.script()
@@ -98,6 +98,10 @@ class BamMetricsTest extends TestNGSuite with Matchers {
 
 object BamMetricsTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+
+  val bam = new File(outputDir, "input" + File.separator + "bla.bam")
+  Files.touch(bam)
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTrait.scala

@@ -89,6 +89,8 @@ trait BastyTrait extends MultiSampleQScript {
     addAll(shiva.functions)
     addSummaryQScript(shiva)
 
+    inputFiles :::= shiva.inputFiles
+
     addSamplesJobs()
   }
 

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -48,6 +48,10 @@ trait BiopetQScript extends Configurable with GatkLogging {
 
   var outputFiles: Map[String, File] = Map()
 
+  type InputFile = BiopetQScript.InputFile
+
+  var inputFiles: List[InputFile] = Nil
+
   /** Get implemented from org.broadinstitute.gatk.queue.QScript */
   var qSettings: QSettings
 
@@ -86,7 +90,16 @@ trait BiopetQScript extends Configurable with GatkLogging {
       globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
     else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, outputdir can not be created")
 
-    reportClass.foreach(add(_))
+    inputFiles.foreach { i =>
+      if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
+      else if (!i.file.canRead()) Logging.addError(s"Input file can not be read: ${i.file}")
+    }
+    
+    this match {
+      case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
+        logger.info("Write report is skipped because sample flag is used")
+      case _ => reportClass.foreach(add(_))
+    }
 
     Logging.checkErrors()
   }
@@ -103,3 +116,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
     add(function)
   }
 }
+
+object BiopetQScript {
+  protected case class InputFile(file: File, md5: Option[String] = None)
+}
\ No newline at end of file

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala

@@ -26,7 +26,7 @@ trait MultiSampleQScript extends SummaryQScript {
   qscript =>
 
   @Argument(doc = "Only Sample", shortName = "s", required = false, fullName = "sample")
-  private val onlySamples: List[String] = Nil
+  private[core] val onlySamples: List[String] = Nil
 
   require(globalConfig.map.contains("samples"), "No Samples found in config")
 
@@ -131,7 +131,7 @@ trait MultiSampleQScript extends SummaryQScript {
 
   /** Runs addAndTrackJobs method for each sample */
   final def addSamplesJobs() {
-    if (onlySamples.isEmpty) {
+    if (onlySamples.isEmpty || samples.forall(x => onlySamples.contains(x._1))) {
       samples.foreach { case (sampleId, sample) => sample.addAndTrackJobs() }
       addMultiSampleJobs()
     } else onlySamples.foreach(sampleId => samples.get(sampleId) match {

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/CheckChecksum.scala

+package nl.lumc.sasc.biopet.core.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.summary.WriteSummary
+import org.broadinstitute.gatk.queue.function.InProcessFunction
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Input }
+
+/**
+ * This class checks md5sums and give an exit code 1 when md5sum is not the same
+ *
+ * Created by pjvanthof on 16/08/15.
+ */
+class CheckChecksum extends InProcessFunction {
+  @Input(required = true)
+  var inputFile: File = _
+
+  @Input(required = true)
+  var checksumFile: File = _
+
+  @Argument(required = true)
+  var checksum: String = _
+
+  override def freezeFieldValues(): Unit = {
+    super.freezeFieldValues()
+    jobOutputFile = new File(checksumFile.getParentFile, checksumFile.getName + ".check.out")
+  }
+
+  /** Exits whenever the input md5sum is not the same as the output md5sum */
+  def run: Unit = {
+    val outputChecksum = WriteSummary.parseChecksum(checksumFile).toLowerCase
+
+    if (outputChecksum != checksum.toLowerCase) {
+      logger.error(s"Input file: '$inputFile' md5sum is not as expected, aborting pipeline")
+
+      // 130 Simulates a ctr-C
+      Runtime.getRuntime.halt(130)
+    }
+  }
+}
\ No newline at end of file

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -231,6 +231,7 @@ object ReportBuilder {
       case Some(template) => template
       case _ =>
         val tempFile = File.createTempFile("ssp-template", new File(location).getName)
+        tempFile.deleteOnExit()
         IoUtils.copyStreamToFile(getClass.getResourceAsStream(location), tempFile)
         templateCache += location -> tempFile
         tempFile

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala

@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.core.summary
 import java.io.File
 
 import nl.lumc.sasc.biopet.core._
-import nl.lumc.sasc.biopet.core.extensions.Md5sum
+import nl.lumc.sasc.biopet.core.extensions.{ CheckChecksum, Md5sum }
 
 import scala.collection.mutable
 
@@ -27,7 +27,7 @@ import scala.collection.mutable
  *
  * Created by pjvan_thof on 2/14/15.
  */
-trait SummaryQScript extends BiopetQScript {
+trait SummaryQScript extends BiopetQScript { qscript =>
 
   /** Key is sample/library, None is sample or library is not applicable */
   private[summary] var summarizables: Map[(String, Option[String], Option[String]), List[Summarizable]] = Map()
@@ -116,6 +116,20 @@ trait SummaryQScript extends BiopetQScript {
       //TODO: add more checksums types
     }
 
+    for (inputFile <- inputFiles) {
+      inputFile.md5 match {
+        case Some(checksum) => {
+          val checkMd5 = new CheckChecksum
+          checkMd5.inputFile = inputFile.file
+          require(SummaryQScript.md5sumCache.contains(inputFile.file), "Md5 job is not executed, checksum file can't be found")
+          checkMd5.checksumFile = SummaryQScript.md5sumCache(inputFile.file)
+          checkMd5.checksum = checksum
+          add(checkMd5)
+        }
+        case _ =>
+      }
+    }
+
     for ((_, summarizableList) <- summarizables; summarizable <- summarizableList) {
       summarizable match {
         case f: BiopetCommandLineFunctionTrait => f.beforeGraph()
@@ -135,7 +149,11 @@ trait SummaryQScript extends BiopetQScript {
     for ((_, file) <- this.summaryFiles)
       addChecksum(file)
 
-    add(writeSummary)
+    this match {
+      case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
+        logger.info("Write summary is skipped because sample flag is used")
+      case _ => add(writeSummary)
+    }
   }
 }
 

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala

@@ -153,10 +153,11 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
   def parseFile(file: File): Map[String, Any] = {
     val map: mutable.Map[String, Any] = mutable.Map()
     map += "path" -> file.getAbsolutePath
-    if (md5sum) map += "md5" -> parseChecksum(SummaryQScript.md5sumCache(file))
+    if (md5sum) map += "md5" -> WriteSummary.parseChecksum(SummaryQScript.md5sumCache(file))
     map.toMap
   }
-
+}
+object WriteSummary {
   /** Retrive checksum from file */
   def parseChecksum(checksumFile: File): String = {
     Source.fromFile(checksumFile).getLines().toList.head.split(" ")(0)

public/biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/MpileupToVcf.scala

@@ -57,7 +57,7 @@ class MpileupToVcf(val root: Configurable) extends ToolCommandFuntion with Refer
   }
 
   override def beforeCmd(): Unit = {
-    if (sample == null && inputBam.exists()) {
+    if (sample == null && inputBam.exists() && inputBam.length() > 0) {
       val inputSam = SamReaderFactory.makeDefault.open(inputBam)
       val readGroups = inputSam.getFileHeader.getReadGroups
       val samples = readGroups.map(readGroup => readGroup.getSample).distinct

public/biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/VcfStats.scala

@@ -625,6 +625,7 @@ object VcfStats extends ToolCommand {
   def executeRscript(resource: String, args: Array[String]): Unit = {
     val is = getClass.getResourceAsStream(resource)
     val file = File.createTempFile("script.", "." + resource)
+    file.deleteOnExit()
     val os = new FileOutputStream(file)
     org.apache.commons.io.IOUtils.copy(is, os)
     os.close()

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/BastyGenerateFastaTest.scala

@@ -31,6 +31,7 @@ class BastyGenerateFastaTest extends TestNGSuite with MockitoSugar with Matchers
 
   @Test def testMainVcf = {
     val tmp = File.createTempFile("basty_out", ".fa")
+    tmp.deleteOnExit()
     val tmppath = tmp.getAbsolutePath
     tmp.deleteOnExit()
 
@@ -40,6 +41,7 @@ class BastyGenerateFastaTest extends TestNGSuite with MockitoSugar with Matchers
 
   @Test def testMainVcfAndBam = {
     val tmp = File.createTempFile("basty_out", ".fa")
+    tmp.deleteOnExit()
     val tmppath = tmp.getAbsolutePath
     tmp.deleteOnExit()
 
@@ -49,6 +51,7 @@ class BastyGenerateFastaTest extends TestNGSuite with MockitoSugar with Matchers
 
   @Test def testMainVcfAndBamMore = {
     val tmp = File.createTempFile("basty_out", ".fa")
+    tmp.deleteOnExit()
     val tmppath = tmp.getAbsolutePath
     tmp.deleteOnExit()
 

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/BiopetFlagstatTest.scala

@@ -56,6 +56,7 @@ class BiopetFlagstatTest extends TestNGSuite with MockitoSugar with Matchers {
   def testMain() = {
     //TODO: Test output file
     val output = File.createTempFile("testMain", ".biopetflagstat")
+    output.deleteOnExit()
     main(Array("-I", bam.getAbsolutePath, "-o", output.toString))
   }
 

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/FastqSplitterTest.scala

@@ -23,7 +23,7 @@ class FastqSplitterTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def testMain() = {
     val temp = File.createTempFile("out", ".fastq")
-
+    temp.deleteOnExit()
     val args = Array("-I", fq, "-o", temp.getAbsolutePath)
     main(args)
   }
@@ -31,6 +31,7 @@ class FastqSplitterTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def testManyOutMain() = {
     val files = (0 until 10).map(_ => File.createTempFile("out", ".fastq"))
+    files.foreach(_.deleteOnExit())
     var args = Array("-I", fq)
     files.foreach(x => args ++= Array("-o", x.getAbsolutePath))
     main(args)

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/FindRepeatsPacBioTest.scala

@@ -29,6 +29,7 @@ class FindRepeatsPacBioTest extends TestNGSuite with MockitoSugar with Matchers
   def testMain() = {
 
     val outputFile = File.createTempFile("repeats", ".tsv")
+    outputFile.deleteOnExit()
     val args = Array("-I", bam, "-b", bed, "-o", outputFile.toString)
     main(args)
   }

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/MpileupToVcfTest.scala

@@ -28,6 +28,7 @@ class MpileupToVcfTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def testMain() = {
     val tmp = File.createTempFile("mpileup", ".vcf")
+    tmp.deleteOnExit()
     val args = Array("-I", pileup, "--sample", "test", "-o", tmp.getAbsolutePath)
 
     main(args)
@@ -36,6 +37,7 @@ class MpileupToVcfTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def validateOutVcf() = {
     val tmp = File.createTempFile("mpileup", ".vcf")
+    tmp.deleteOnExit()
     val args = Array("-I", pileup, "--sample", "test", "-o", tmp.getAbsolutePath, "--minDP", "1", "--minAP", "1")
     main(args)
 
@@ -51,6 +53,7 @@ class MpileupToVcfTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def extraValidateOutVcf() = {
     val tmp = File.createTempFile("mpileup", ".vcf")
+    tmp.deleteOnExit()
     val args = Array("-I", pileup, "--sample", "test", "-o", tmp.getAbsolutePath, "--minDP", "1", "--minAP", "1")
     main(args)
 

public/biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/PrefixFastqTest.scala

@@ -26,6 +26,7 @@ class PrefixFastqTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def testMain() = {
     val temp = File.createTempFile("out", ".fastq")
+    temp.deleteOnExit()
 
     val args = Array("-i", fq, "-o", temp.getAbsolutePath, "-s", "AAA")
     main(args)
@@ -34,6 +35,7 @@ class PrefixFastqTest extends TestNGSuite with MockitoSugar with Matchers {
   @Test
   def testOutput() = {
     val temp = File.createTempFile("out", ".fastq")
+    temp.deleteOnExit()
 
     val args = Array("-i", fq, "-o", temp.getAbsolutePath, "-s", "AAA")
     main(args)