Merge branch 'feature-input_files_checking' into develop

Conflicts:
	public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala

@@ -93,6 +93,12 @@ class ShivaTest extends TestNGSuite with Matchers {
 
 object ShivaTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)
@@ -136,8 +142,8 @@ object ShivaTest {
   val sample1 = Map(
     "samples" -> Map("sample1" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "1_1_R1.fq",
-        "R2" -> "1_1_R2.fq"
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
       )
     )
     )))
@@ -145,8 +151,8 @@ object ShivaTest {
   val sample2 = Map(
     "samples" -> Map("sample2" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "2_1_R1.fq",
-        "R2" -> "2_1_R2.fq"
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
       )
     )
     )))
@@ -154,12 +160,12 @@ object ShivaTest {
   val sample3 = Map(
     "samples" -> Map("sample3" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "3_1_R1.fq",
-        "R2" -> "3_1_R2.fq"
+        "R1" -> inputTouch("3_1_R1.fq"),
+        "R2" -> inputTouch("3_1_R2.fq")
       ),
       "lib2" -> Map(
-        "R1" -> "3_2_R1.fq",
-        "R2" -> "3_2_R2.fq"
+        "R1" -> inputTouch("3_2_R1.fq"),
+        "R2" -> inputTouch("3_2_R2.fq")
       )
     )
     )))

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala

@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
     val map = Map("variantcallers" -> callers.toList)
     val pipeline = initPipeline(map)
 
-    pipeline.inputBams = (for (n <- 1 to bams) yield new File("bam_" + n + ".bam")).toList
+    pipeline.inputBams = (for (n <- 1 to bams) yield ShivaVariantcallingTest.inputTouch("bam_" + n + ".bam")).toList
 
     val illegalArgumentException = pipeline.inputBams.isEmpty ||
       (!raw && !bcftools &&
@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
 
 object ShivaVariantcallingTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): File = {
+    val file = new File(outputDir, "input" + File.separator + name).getAbsoluteFile
+    Files.touch(file)
+    file
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/bam2wig/src/main/scala/nl/lumc/sasc/biopet/pipelines/bamtobigwig/Bam2Wig.scala

@@ -35,6 +35,7 @@ class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript {
   var bamFile: File = null
 
   def init(): Unit = {
+    inputFiles :+= new InputFile(bamFile)
   }
 
   def biopetScript(): Unit = {

public/bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetrics.scala

@@ -71,7 +71,8 @@ class BamMetrics(val root: Configurable) extends QScript with SummaryQScript wit
   }
 
   /** executed before script */
-  def init() {
+  def init(): Unit = {
+    inputFiles :+= new InputFile(inputBam)
   }
 
   /** Script to add jobs */

public/bammetrics/src/test/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetricsTest.scala

@@ -69,7 +69,7 @@ class BamMetricsTest extends TestNGSuite with Matchers {
       Map("regions_of_interest" -> (1 to rois).map("roi_" + _ + ".bed").toList)
     val bammetrics: BamMetrics = initPipeline(map)
 
-    bammetrics.inputBam = new File("input.bam")
+    bammetrics.inputBam = BamMetricsTest.bam
     bammetrics.sampleId = Some("1")
     bammetrics.libId = Some("1")
     bammetrics.script()
@@ -98,6 +98,10 @@ class BamMetricsTest extends TestNGSuite with Matchers {
 
 object BamMetricsTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+
+  val bam = new File(outputDir, "input" + File.separator + "bla.bam")
+  Files.touch(bam)
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTrait.scala

@@ -89,6 +89,8 @@ trait BastyTrait extends MultiSampleQScript {
     addAll(shiva.functions)
     addSummaryQScript(shiva)
 
+    inputFiles :::= shiva.inputFiles
+
     addSamplesJobs()
   }
 

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -48,6 +48,10 @@ trait BiopetQScript extends Configurable with GatkLogging {
 
   var outputFiles: Map[String, File] = Map()
 
+  type InputFile = BiopetQScript.InputFile
+
+  var inputFiles: List[InputFile] = Nil
+
   /** Get implemented from org.broadinstitute.gatk.queue.QScript */
   var qSettings: QSettings
 
@@ -86,6 +90,11 @@ trait BiopetQScript extends Configurable with GatkLogging {
       globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
     else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, outputdir can not be created")
 
+    inputFiles.foreach { i =>
+      if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
+      else if (!i.file.canRead()) Logging.addError(s"Input file can not be read: ${i.file}")
+    }
+    
     this match {
       case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
         logger.info("Write report is skipped because sample flag is used")
@@ -107,3 +116,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
     add(function)
   }
 }
+
+object BiopetQScript {
+  protected case class InputFile(file: File, md5: Option[String] = None)
+}
\ No newline at end of file

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/CheckChecksum.scala

+package nl.lumc.sasc.biopet.core.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.summary.WriteSummary
+import org.broadinstitute.gatk.queue.function.InProcessFunction
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Input }
+
+/**
+ * This class checks md5sums and give an exit code 1 when md5sum is not the same
+ *
+ * Created by pjvanthof on 16/08/15.
+ */
+class CheckChecksum extends InProcessFunction {
+  @Input(required = true)
+  var inputFile: File = _
+
+  @Input(required = true)
+  var checksumFile: File = _
+
+  @Argument(required = true)
+  var checksum: String = _
+
+  override def freezeFieldValues(): Unit = {
+    super.freezeFieldValues()
+    jobOutputFile = new File(checksumFile.getParentFile, checksumFile.getName + ".check.out")
+  }
+
+  /** Exits whenever the input md5sum is not the same as the output md5sum */
+  def run: Unit = {
+    val outputChecksum = WriteSummary.parseChecksum(checksumFile).toLowerCase
+
+    if (outputChecksum != checksum.toLowerCase) {
+      logger.error(s"Input file: '$inputFile' md5sum is not as expected, aborting pipeline")
+
+      // 130 Simulates a ctr-C
+      Runtime.getRuntime.halt(130)
+    }
+  }
+}
\ No newline at end of file

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala

@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.core.summary
 import java.io.File
 
 import nl.lumc.sasc.biopet.core._
-import nl.lumc.sasc.biopet.core.extensions.Md5sum
+import nl.lumc.sasc.biopet.core.extensions.{ CheckChecksum, Md5sum }
 
 import scala.collection.mutable
 
@@ -27,7 +27,7 @@ import scala.collection.mutable
  *
  * Created by pjvan_thof on 2/14/15.
  */
-trait SummaryQScript extends BiopetQScript {
+trait SummaryQScript extends BiopetQScript { qscript =>
 
   /** Key is sample/library, None is sample or library is not applicable */
   private[summary] var summarizables: Map[(String, Option[String], Option[String]), List[Summarizable]] = Map()
@@ -116,6 +116,20 @@ trait SummaryQScript extends BiopetQScript {
       //TODO: add more checksums types
     }
 
+    for (inputFile <- inputFiles) {
+      inputFile.md5 match {
+        case Some(checksum) => {
+          val checkMd5 = new CheckChecksum
+          checkMd5.inputFile = inputFile.file
+          require(SummaryQScript.md5sumCache.contains(inputFile.file), "Md5 job is not executed, checksum file can't be found")
+          checkMd5.checksumFile = SummaryQScript.md5sumCache(inputFile.file)
+          checkMd5.checksum = checksum
+          add(checkMd5)
+        }
+        case _ =>
+      }
+    }
+
     for ((_, summarizableList) <- summarizables; summarizable <- summarizableList) {
       summarizable match {
         case f: BiopetCommandLineFunctionTrait => f.beforeGraph()

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala

@@ -153,10 +153,11 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
   def parseFile(file: File): Map[String, Any] = {
     val map: mutable.Map[String, Any] = mutable.Map()
     map += "path" -> file.getAbsolutePath
-    if (md5sum) map += "md5" -> parseChecksum(SummaryQScript.md5sumCache(file))
+    if (md5sum) map += "md5" -> WriteSummary.parseChecksum(SummaryQScript.md5sumCache(file))
     map.toMap
   }
-
+}
+object WriteSummary {
   /** Retrive checksum from file */
   def parseChecksum(checksumFile: File): String = {
     Source.fromFile(checksumFile).getLines().toList.head.split(" ")(0)

public/biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/MpileupToVcf.scala

@@ -57,7 +57,7 @@ class MpileupToVcf(val root: Configurable) extends ToolCommandFuntion with Refer
   }
 
   override def beforeCmd(): Unit = {
-    if (sample == null && inputBam.exists()) {
+    if (sample == null && inputBam.exists() && inputBam.length() > 0) {
       val inputSam = SamReaderFactory.makeDefault.open(inputBam)
       val readGroups = inputSam.getFileHeader.getReadGroups
       val samples = readGroups.map(readGroup => readGroup.getSample).distinct

public/carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala

@@ -78,6 +78,10 @@ class Carp(val root: Configurable) extends QScript with MultiSampleQScript with
         if (config.contains("R1")) {
           mapping.input_R1 = config("R1")
           if (config.contains("R2")) mapping.input_R2 = config("R2")
+
+          inputFiles :+= new InputFile(mapping.input_R1, config("R1_md5"))
+          mapping.input_R2.foreach(inputFiles :+= new InputFile(_, config("R2_md5")))
+
           mapping.init()
           mapping.biopetScript()
           addAll(mapping.functions)

public/carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala

@@ -97,6 +97,12 @@ class CarpTest extends TestNGSuite with Matchers {
 
 object CarpTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)
@@ -127,8 +133,8 @@ object CarpTest {
   val sample1 = Map(
     "samples" -> Map("sample1" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "1_1_R1.fq",
-        "R2" -> "1_1_R2.fq"
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
       )
     )
     )))
@@ -136,8 +142,8 @@ object CarpTest {
   val sample2 = Map(
     "samples" -> Map("sample2" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "2_1_R1.fq",
-        "R2" -> "2_1_R2.fq"
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
       )
     )
     )))
@@ -145,12 +151,12 @@ object CarpTest {
   val sample3 = Map(
     "samples" -> Map("sample3" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "3_1_R1.fq",
-        "R2" -> "3_1_R2.fq"
+        "R1" -> inputTouch("3_1_R1.fq"),
+        "R2" -> inputTouch("3_1_R2.fq")
       ),
       "lib2" -> Map(
-        "R1" -> "3_2_R1.fq",
-        "R2" -> "3_2_R2.fq"
+        "R1" -> inputTouch("3_2_R1.fq"),
+        "R2" -> inputTouch("3_2_R2.fq")
       )
     )
     )))
@@ -158,8 +164,8 @@ object CarpTest {
   val threatment1 = Map(
     "samples" -> Map("threatment" -> Map("control" -> "control1", "libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "threatment_1_R1.fq",
-        "R2" -> "threatment_1_R2.fq"
+        "R1" -> inputTouch("threatment_1_R1.fq"),
+        "R2" -> inputTouch("threatment_1_R2.fq")
       )
     )
     )))
@@ -167,8 +173,8 @@ object CarpTest {
   val control1 = Map(
     "samples" -> Map("control1" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "control_1_R1.fq",
-        "R2" -> "control_1_R2.fq"
+        "R1" -> inputTouch("control_1_R1.fq"),
+        "R2" -> inputTouch("control_1_R2.fq")
       )
     )
     )))

public/flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala

@@ -85,6 +85,9 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
 
     paired = input_R2.isDefined
 
+    inputFiles :+= new InputFile(input_R1)
+    input_R2.foreach(inputFiles :+= new InputFile(_))
+
     if (input_R1.endsWith(".gz")) R1_name = input_R1.getName.substring(0, input_R1.getName.lastIndexOf(".gz"))
     else if (input_R1.endsWith(".gzip")) R1_name = input_R1.getName.substring(0, input_R1.getName.lastIndexOf(".gzip"))
     else R1_name = input_R1.getName

public/flexiprep/src/test/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/FlexiprepTest.scala

@@ -67,8 +67,8 @@ class FlexiprepTest extends TestNGSuite with Matchers {
     ), Map(FlexiprepTest.executables.toSeq: _*))
     val flexiprep: Flexiprep = initPipeline(map)
 
-    flexiprep.input_R1 = new File(flexiprep.outputDir, "bla_R1.fq" + (if (zipped) ".gz" else ""))
-    if (paired) flexiprep.input_R2 = Some(new File(flexiprep.outputDir, "bla_R2.fq" + (if (zipped) ".gz" else "")))
+    flexiprep.input_R1 = (if (zipped) FlexiprepTest.r1Zipped else FlexiprepTest.r1)
+    if (paired) flexiprep.input_R2 = Some((if (zipped) FlexiprepTest.r2Zipped else FlexiprepTest.r2))
     flexiprep.sampleId = Some("1")
     flexiprep.libId = Some("1")
     flexiprep.script()
@@ -95,6 +95,16 @@ class FlexiprepTest extends TestNGSuite with Matchers {
 
 object FlexiprepTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+
+  val r1 = new File(outputDir, "input" + File.separator + "R1.fq")
+  Files.touch(r1)
+  val r2 = new File(outputDir, "input" + File.separator + "R2.fq")
+  Files.touch(r2)
+  val r1Zipped = new File(outputDir, "input" + File.separator + "R1.fq.gz")
+  Files.touch(r1Zipped)
+  val r2Zipped = new File(outputDir, "input" + File.separator + "R2.fq.gz")
+  Files.touch(r2Zipped)
 
   val executables = Map(
     "seqstat" -> Map("exe" -> "test"),

public/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala

@@ -844,6 +844,8 @@ class Gentrap(val root: Configurable) extends QScript
       def addJobs(): Unit = {
         // create per-library alignment file
         addAll(mappingJob.functions)
+        // Input file checking
+        inputFiles :::= mappingJob.inputFiles
         // add bigwig track
         addAll(bam2wigModule.functions)
         qscript.addSummaryQScript(mappingJob)

public/gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala

@@ -43,8 +43,8 @@ class GentrapTest extends TestNGSuite with Matchers {
 
   /** Convenience method for making library config */
   private def makeLibConfig(idx: Int, paired: Boolean = true) = {
-    val files = Map("R1" -> "test_R1.fq")
-    if (paired) (s"lib_$idx", files ++ Map("R2" -> "test_R2.fq"))
+    val files = Map("R1" -> GentrapTest.inputTouch("test_R1.fq"))
+    if (paired) (s"lib_$idx", files ++ Map("R2" -> GentrapTest.inputTouch("test_R2.fq")))
     else (s"lib_$idx", files)
   }
 
@@ -179,6 +179,12 @@ class GentrapTest extends TestNGSuite with Matchers {
 
 object GentrapTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/Mapping.scala

@@ -137,6 +137,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
     require(sampleId.isDefined, "Missing sample ID on mapping module")
     require(libId.isDefined, "Missing library ID on mapping module")
 
+    inputFiles :+= new InputFile(input_R1)
+    input_R2.foreach(inputFiles :+= new InputFile(_))
+
     paired = input_R2.isDefined
 
     if (readgroupId == null) readgroupId = sampleId.get + "-" + libId.get

public/mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MappingTest.scala

@@ -79,11 +79,11 @@ class MappingTest extends TestNGSuite with Matchers {
     val mapping: Mapping = initPipeline(map)
 
     if (zipped) {
-      mapping.input_R1 = new File(mapping.outputDir, "bla_R1.fq.gz")
-      if (paired) mapping.input_R2 = Some(new File(mapping.outputDir, "bla_R2.fq.gz"))
+      mapping.input_R1 = MappingTest.r1Zipped
+      if (paired) mapping.input_R2 = Some(MappingTest.r2Zipped)
     } else {
-      mapping.input_R1 = new File(mapping.outputDir, "bla_R1.fq")
-      if (paired) mapping.input_R2 = Some(new File(mapping.outputDir, "bla_R2.fq"))
+      mapping.input_R1 = MappingTest.r1
+      if (paired) mapping.input_R2 = Some(MappingTest.r2)
     }
     mapping.sampleId = Some("1")
     mapping.libId = Some("1")
@@ -131,6 +131,16 @@ class MappingTest extends TestNGSuite with Matchers {
 
 object MappingTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+
+  val r1 = new File(outputDir, "input" + File.separator + "R1.fq")
+  Files.touch(r1)
+  val r2 = new File(outputDir, "input" + File.separator + "R2.fq")
+  Files.touch(r2)
+  val r1Zipped = new File(outputDir, "input" + File.separator + "R1.fq.gz")
+  Files.touch(r1Zipped)
+  val r2Zipped = new File(outputDir, "input" + File.separator + "R2.fq.gz")
+  Files.touch(r2Zipped)
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

public/sage/src/main/scala/nl/lumc/sasc/biopet/pipelines/sage/Sage.scala

@@ -88,6 +88,8 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
       mapping.sampleId = Some(sampleId)
 
       protected def addJobs(): Unit = {
+        inputFiles :+= new InputFile(inputFastq, config("R1_md5"))
+
         flexiprep.outputDir = new File(libDir, "flexiprep/")
         flexiprep.input_R1 = inputFastq
         flexiprep.init()