Commit f4063de9 authored by Sander Bollen's avatar Sander Bollen
Browse files

Merge branch 'feature-input_files_checking' into develop

Conflicts:
	public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala
parents 92170802 6c7f4701
......@@ -93,6 +93,12 @@ class ShivaTest extends TestNGSuite with Matchers {
object ShivaTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): String = {
val file = new File(outputDir, "input" + File.separator + name)
Files.touch(file)
file.getAbsolutePath
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......@@ -136,8 +142,8 @@ object ShivaTest {
val sample1 = Map(
"samples" -> Map("sample1" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "1_1_R1.fq",
"R2" -> "1_1_R2.fq"
"R1" -> inputTouch("1_1_R1.fq"),
"R2" -> inputTouch("1_1_R2.fq")
)
)
)))
......@@ -145,8 +151,8 @@ object ShivaTest {
val sample2 = Map(
"samples" -> Map("sample2" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "2_1_R1.fq",
"R2" -> "2_1_R2.fq"
"R1" -> inputTouch("2_1_R1.fq"),
"R2" -> inputTouch("2_1_R2.fq")
)
)
)))
......@@ -154,12 +160,12 @@ object ShivaTest {
val sample3 = Map(
"samples" -> Map("sample3" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "3_1_R1.fq",
"R2" -> "3_1_R2.fq"
"R1" -> inputTouch("3_1_R1.fq"),
"R2" -> inputTouch("3_1_R2.fq")
),
"lib2" -> Map(
"R1" -> "3_2_R1.fq",
"R2" -> "3_2_R2.fq"
"R1" -> inputTouch("3_2_R1.fq"),
"R2" -> inputTouch("3_2_R2.fq")
)
)
)))
......
......@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
val map = Map("variantcallers" -> callers.toList)
val pipeline = initPipeline(map)
pipeline.inputBams = (for (n <- 1 to bams) yield new File("bam_" + n + ".bam")).toList
pipeline.inputBams = (for (n <- 1 to bams) yield ShivaVariantcallingTest.inputTouch("bam_" + n + ".bam")).toList
val illegalArgumentException = pipeline.inputBams.isEmpty ||
(!raw && !bcftools &&
......@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
object ShivaVariantcallingTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): File = {
val file = new File(outputDir, "input" + File.separator + name).getAbsoluteFile
Files.touch(file)
file
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......
......@@ -35,6 +35,7 @@ class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript {
var bamFile: File = null
def init(): Unit = {
inputFiles :+= new InputFile(bamFile)
}
def biopetScript(): Unit = {
......
......@@ -71,7 +71,8 @@ class BamMetrics(val root: Configurable) extends QScript with SummaryQScript wit
}
/** executed before script */
def init() {
def init(): Unit = {
inputFiles :+= new InputFile(inputBam)
}
/** Script to add jobs */
......
......@@ -69,7 +69,7 @@ class BamMetricsTest extends TestNGSuite with Matchers {
Map("regions_of_interest" -> (1 to rois).map("roi_" + _ + ".bed").toList)
val bammetrics: BamMetrics = initPipeline(map)
bammetrics.inputBam = new File("input.bam")
bammetrics.inputBam = BamMetricsTest.bam
bammetrics.sampleId = Some("1")
bammetrics.libId = Some("1")
bammetrics.script()
......@@ -98,6 +98,10 @@ class BamMetricsTest extends TestNGSuite with Matchers {
object BamMetricsTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
val bam = new File(outputDir, "input" + File.separator + "bla.bam")
Files.touch(bam)
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......
......@@ -89,6 +89,8 @@ trait BastyTrait extends MultiSampleQScript {
addAll(shiva.functions)
addSummaryQScript(shiva)
inputFiles :::= shiva.inputFiles
addSamplesJobs()
}
......
......@@ -48,6 +48,10 @@ trait BiopetQScript extends Configurable with GatkLogging {
var outputFiles: Map[String, File] = Map()
type InputFile = BiopetQScript.InputFile
var inputFiles: List[InputFile] = Nil
/** Get implemented from org.broadinstitute.gatk.queue.QScript */
var qSettings: QSettings
......@@ -86,6 +90,11 @@ trait BiopetQScript extends Configurable with GatkLogging {
globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, outputdir can not be created")
inputFiles.foreach { i =>
if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
else if (!i.file.canRead()) Logging.addError(s"Input file can not be read: ${i.file}")
}
this match {
case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
logger.info("Write report is skipped because sample flag is used")
......@@ -107,3 +116,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
add(function)
}
}
object BiopetQScript {
protected case class InputFile(file: File, md5: Option[String] = None)
}
\ No newline at end of file
package nl.lumc.sasc.biopet.core.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.summary.WriteSummary
import org.broadinstitute.gatk.queue.function.InProcessFunction
import org.broadinstitute.gatk.utils.commandline.{ Argument, Input }
/**
* This class checks md5sums and give an exit code 1 when md5sum is not the same
*
* Created by pjvanthof on 16/08/15.
*/
class CheckChecksum extends InProcessFunction {
@Input(required = true)
var inputFile: File = _
@Input(required = true)
var checksumFile: File = _
@Argument(required = true)
var checksum: String = _
override def freezeFieldValues(): Unit = {
super.freezeFieldValues()
jobOutputFile = new File(checksumFile.getParentFile, checksumFile.getName + ".check.out")
}
/** Exits whenever the input md5sum is not the same as the output md5sum */
def run: Unit = {
val outputChecksum = WriteSummary.parseChecksum(checksumFile).toLowerCase
if (outputChecksum != checksum.toLowerCase) {
logger.error(s"Input file: '$inputFile' md5sum is not as expected, aborting pipeline")
// 130 Simulates a ctr-C
Runtime.getRuntime.halt(130)
}
}
}
\ No newline at end of file
......@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.core.summary
import java.io.File
import nl.lumc.sasc.biopet.core._
import nl.lumc.sasc.biopet.core.extensions.Md5sum
import nl.lumc.sasc.biopet.core.extensions.{ CheckChecksum, Md5sum }
import scala.collection.mutable
......@@ -27,7 +27,7 @@ import scala.collection.mutable
*
* Created by pjvan_thof on 2/14/15.
*/
trait SummaryQScript extends BiopetQScript {
trait SummaryQScript extends BiopetQScript { qscript =>
/** Key is sample/library, None is sample or library is not applicable */
private[summary] var summarizables: Map[(String, Option[String], Option[String]), List[Summarizable]] = Map()
......@@ -116,6 +116,20 @@ trait SummaryQScript extends BiopetQScript {
//TODO: add more checksums types
}
for (inputFile <- inputFiles) {
inputFile.md5 match {
case Some(checksum) => {
val checkMd5 = new CheckChecksum
checkMd5.inputFile = inputFile.file
require(SummaryQScript.md5sumCache.contains(inputFile.file), "Md5 job is not executed, checksum file can't be found")
checkMd5.checksumFile = SummaryQScript.md5sumCache(inputFile.file)
checkMd5.checksum = checksum
add(checkMd5)
}
case _ =>
}
}
for ((_, summarizableList) <- summarizables; summarizable <- summarizableList) {
summarizable match {
case f: BiopetCommandLineFunctionTrait => f.beforeGraph()
......
......@@ -153,10 +153,11 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
def parseFile(file: File): Map[String, Any] = {
val map: mutable.Map[String, Any] = mutable.Map()
map += "path" -> file.getAbsolutePath
if (md5sum) map += "md5" -> parseChecksum(SummaryQScript.md5sumCache(file))
if (md5sum) map += "md5" -> WriteSummary.parseChecksum(SummaryQScript.md5sumCache(file))
map.toMap
}
}
object WriteSummary {
/** Retrive checksum from file */
def parseChecksum(checksumFile: File): String = {
Source.fromFile(checksumFile).getLines().toList.head.split(" ")(0)
......
......@@ -57,7 +57,7 @@ class MpileupToVcf(val root: Configurable) extends ToolCommandFuntion with Refer
}
override def beforeCmd(): Unit = {
if (sample == null && inputBam.exists()) {
if (sample == null && inputBam.exists() && inputBam.length() > 0) {
val inputSam = SamReaderFactory.makeDefault.open(inputBam)
val readGroups = inputSam.getFileHeader.getReadGroups
val samples = readGroups.map(readGroup => readGroup.getSample).distinct
......
......@@ -78,6 +78,10 @@ class Carp(val root: Configurable) extends QScript with MultiSampleQScript with
if (config.contains("R1")) {
mapping.input_R1 = config("R1")
if (config.contains("R2")) mapping.input_R2 = config("R2")
inputFiles :+= new InputFile(mapping.input_R1, config("R1_md5"))
mapping.input_R2.foreach(inputFiles :+= new InputFile(_, config("R2_md5")))
mapping.init()
mapping.biopetScript()
addAll(mapping.functions)
......
......@@ -97,6 +97,12 @@ class CarpTest extends TestNGSuite with Matchers {
object CarpTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): String = {
val file = new File(outputDir, "input" + File.separator + name)
Files.touch(file)
file.getAbsolutePath
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......@@ -127,8 +133,8 @@ object CarpTest {
val sample1 = Map(
"samples" -> Map("sample1" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "1_1_R1.fq",
"R2" -> "1_1_R2.fq"
"R1" -> inputTouch("1_1_R1.fq"),
"R2" -> inputTouch("1_1_R2.fq")
)
)
)))
......@@ -136,8 +142,8 @@ object CarpTest {
val sample2 = Map(
"samples" -> Map("sample2" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "2_1_R1.fq",
"R2" -> "2_1_R2.fq"
"R1" -> inputTouch("2_1_R1.fq"),
"R2" -> inputTouch("2_1_R2.fq")
)
)
)))
......@@ -145,12 +151,12 @@ object CarpTest {
val sample3 = Map(
"samples" -> Map("sample3" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "3_1_R1.fq",
"R2" -> "3_1_R2.fq"
"R1" -> inputTouch("3_1_R1.fq"),
"R2" -> inputTouch("3_1_R2.fq")
),
"lib2" -> Map(
"R1" -> "3_2_R1.fq",
"R2" -> "3_2_R2.fq"
"R1" -> inputTouch("3_2_R1.fq"),
"R2" -> inputTouch("3_2_R2.fq")
)
)
)))
......@@ -158,8 +164,8 @@ object CarpTest {
val threatment1 = Map(
"samples" -> Map("threatment" -> Map("control" -> "control1", "libraries" -> Map(
"lib1" -> Map(
"R1" -> "threatment_1_R1.fq",
"R2" -> "threatment_1_R2.fq"
"R1" -> inputTouch("threatment_1_R1.fq"),
"R2" -> inputTouch("threatment_1_R2.fq")
)
)
)))
......@@ -167,8 +173,8 @@ object CarpTest {
val control1 = Map(
"samples" -> Map("control1" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "control_1_R1.fq",
"R2" -> "control_1_R2.fq"
"R1" -> inputTouch("control_1_R1.fq"),
"R2" -> inputTouch("control_1_R2.fq")
)
)
)))
......
......@@ -85,6 +85,9 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
paired = input_R2.isDefined
inputFiles :+= new InputFile(input_R1)
input_R2.foreach(inputFiles :+= new InputFile(_))
if (input_R1.endsWith(".gz")) R1_name = input_R1.getName.substring(0, input_R1.getName.lastIndexOf(".gz"))
else if (input_R1.endsWith(".gzip")) R1_name = input_R1.getName.substring(0, input_R1.getName.lastIndexOf(".gzip"))
else R1_name = input_R1.getName
......
......@@ -67,8 +67,8 @@ class FlexiprepTest extends TestNGSuite with Matchers {
), Map(FlexiprepTest.executables.toSeq: _*))
val flexiprep: Flexiprep = initPipeline(map)
flexiprep.input_R1 = new File(flexiprep.outputDir, "bla_R1.fq" + (if (zipped) ".gz" else ""))
if (paired) flexiprep.input_R2 = Some(new File(flexiprep.outputDir, "bla_R2.fq" + (if (zipped) ".gz" else "")))
flexiprep.input_R1 = (if (zipped) FlexiprepTest.r1Zipped else FlexiprepTest.r1)
if (paired) flexiprep.input_R2 = Some((if (zipped) FlexiprepTest.r2Zipped else FlexiprepTest.r2))
flexiprep.sampleId = Some("1")
flexiprep.libId = Some("1")
flexiprep.script()
......@@ -95,6 +95,16 @@ class FlexiprepTest extends TestNGSuite with Matchers {
object FlexiprepTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
val r1 = new File(outputDir, "input" + File.separator + "R1.fq")
Files.touch(r1)
val r2 = new File(outputDir, "input" + File.separator + "R2.fq")
Files.touch(r2)
val r1Zipped = new File(outputDir, "input" + File.separator + "R1.fq.gz")
Files.touch(r1Zipped)
val r2Zipped = new File(outputDir, "input" + File.separator + "R2.fq.gz")
Files.touch(r2Zipped)
val executables = Map(
"seqstat" -> Map("exe" -> "test"),
......
......@@ -844,6 +844,8 @@ class Gentrap(val root: Configurable) extends QScript
def addJobs(): Unit = {
// create per-library alignment file
addAll(mappingJob.functions)
// Input file checking
inputFiles :::= mappingJob.inputFiles
// add bigwig track
addAll(bam2wigModule.functions)
qscript.addSummaryQScript(mappingJob)
......
......@@ -43,8 +43,8 @@ class GentrapTest extends TestNGSuite with Matchers {
/** Convenience method for making library config */
private def makeLibConfig(idx: Int, paired: Boolean = true) = {
val files = Map("R1" -> "test_R1.fq")
if (paired) (s"lib_$idx", files ++ Map("R2" -> "test_R2.fq"))
val files = Map("R1" -> GentrapTest.inputTouch("test_R1.fq"))
if (paired) (s"lib_$idx", files ++ Map("R2" -> GentrapTest.inputTouch("test_R2.fq")))
else (s"lib_$idx", files)
}
......@@ -179,6 +179,12 @@ class GentrapTest extends TestNGSuite with Matchers {
object GentrapTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): String = {
val file = new File(outputDir, "input" + File.separator + name)
Files.touch(file)
file.getAbsolutePath
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......
......@@ -137,6 +137,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
require(sampleId.isDefined, "Missing sample ID on mapping module")
require(libId.isDefined, "Missing library ID on mapping module")
inputFiles :+= new InputFile(input_R1)
input_R2.foreach(inputFiles :+= new InputFile(_))
paired = input_R2.isDefined
if (readgroupId == null) readgroupId = sampleId.get + "-" + libId.get
......
......@@ -79,11 +79,11 @@ class MappingTest extends TestNGSuite with Matchers {
val mapping: Mapping = initPipeline(map)
if (zipped) {
mapping.input_R1 = new File(mapping.outputDir, "bla_R1.fq.gz")
if (paired) mapping.input_R2 = Some(new File(mapping.outputDir, "bla_R2.fq.gz"))
mapping.input_R1 = MappingTest.r1Zipped
if (paired) mapping.input_R2 = Some(MappingTest.r2Zipped)
} else {
mapping.input_R1 = new File(mapping.outputDir, "bla_R1.fq")
if (paired) mapping.input_R2 = Some(new File(mapping.outputDir, "bla_R2.fq"))
mapping.input_R1 = MappingTest.r1
if (paired) mapping.input_R2 = Some(MappingTest.r2)
}
mapping.sampleId = Some("1")
mapping.libId = Some("1")
......@@ -131,6 +131,16 @@ class MappingTest extends TestNGSuite with Matchers {
object MappingTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
val r1 = new File(outputDir, "input" + File.separator + "R1.fq")
Files.touch(r1)
val r2 = new File(outputDir, "input" + File.separator + "R2.fq")
Files.touch(r2)
val r1Zipped = new File(outputDir, "input" + File.separator + "R1.fq.gz")
Files.touch(r1Zipped)
val r2Zipped = new File(outputDir, "input" + File.separator + "R2.fq.gz")
Files.touch(r2Zipped)
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......
......@@ -88,6 +88,8 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
mapping.sampleId = Some(sampleId)
protected def addJobs(): Unit = {
inputFiles :+= new InputFile(inputFastq, config("R1_md5"))
flexiprep.outputDir = new File(libDir, "flexiprep/")
flexiprep.input_R1 = inputFastq
flexiprep.init()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment