Commit f4f1f4cb authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Fix chunking

parent aca99bf8
...@@ -3,7 +3,7 @@ package nl.lumc.sasc.biopet.extensions.samtools ...@@ -3,7 +3,7 @@ package nl.lumc.sasc.biopet.extensions.samtools
import java.io.File import java.io.File
import nl.lumc.sasc.biopet.utils.config.Configurable import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{Input, Output} import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
/** /**
* Created by pjvanthof on 22/09/15. * Created by pjvanthof on 22/09/15.
......
...@@ -153,29 +153,31 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with ...@@ -153,29 +153,31 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
else None else None
/** Adds all chunkable jobs of flexiprep */ /** Adds all chunkable jobs of flexiprep */
def runTrimClip(R1_in: File, outDir: File, chunk: String): (File, Option[File], List[File]) = def runTrimClip(R1_in: File, outDir: File, chunk: String): (File, Option[File]) =
runTrimClip(R1_in, None, outDir, chunk) runTrimClip(R1_in, None, outDir, chunk)
/** Adds all chunkable jobs of flexiprep */ /** Adds all chunkable jobs of flexiprep */
def runTrimClip(R1_in: File, outDir: File): (File, Option[File], List[File]) = def runTrimClip(R1_in: File, outDir: File): (File, Option[File]) =
runTrimClip(R1_in, None, outDir, "") runTrimClip(R1_in, None, outDir, "")
/** Adds all chunkable jobs of flexiprep */ /** Adds all chunkable jobs of flexiprep */
def runTrimClip(R1_in: File, R2_in: Option[File], outDir: File): (File, Option[File], List[File]) = def runTrimClip(R1_in: File, R2_in: Option[File], outDir: File): (File, Option[File]) =
runTrimClip(R1_in, R2_in, outDir, "") runTrimClip(R1_in, R2_in, outDir, "")
/** Adds all chunkable jobs of flexiprep */ /** Adds all chunkable jobs of flexiprep */
def runTrimClip(R1_in: File, R2_in: Option[File], outDir: File, chunkarg: String): (File, Option[File], List[File]) = { def runTrimClip(R1_in: File,
R2_in: Option[File],
outDir: File,
chunkarg: String): (File, Option[File]) = {
val chunk = if (chunkarg.isEmpty || chunkarg.endsWith("_")) chunkarg else chunkarg + "_" val chunk = if (chunkarg.isEmpty || chunkarg.endsWith("_")) chunkarg else chunkarg + "_"
var R1 = R1_in var R1 = R1_in
var R2 = R2_in var R2 = R2_in
def deps: List[File] = Nil
val qcCmdR1 = new QcCommand(this, fastqc_R1) val qcCmdR1 = new QcCommand(this, fastqc_R1)
qcCmdR1.input = R1_in qcCmdR1.input = R1_in
qcCmdR1.read = "R1" qcCmdR1.read = "R1"
qcCmdR1.output = if (paired) new File(fastqR1Qc.getAbsolutePath.stripSuffix(".gz")) qcCmdR1.output = if (paired) new File(outDir, fastqR1Qc.getName.stripSuffix(".gz"))
else fastqR1Qc else fastqR1Qc
qcCmdR1.deps :+= fastqc_R1.output qcCmdR1.deps :+= fastqc_R1.output
qcCmdR1.isIntermediate = paired || !keepQcFastqFiles qcCmdR1.isIntermediate = paired || !keepQcFastqFiles
...@@ -184,7 +186,7 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with ...@@ -184,7 +186,7 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
if (paired) { if (paired) {
val qcCmdR2 = new QcCommand(this, fastqc_R2) val qcCmdR2 = new QcCommand(this, fastqc_R2)
qcCmdR2.input = R2_in.get qcCmdR2.input = R2_in.get
qcCmdR2.output = new File(fastqR2Qc.get.getAbsolutePath.stripSuffix(".gz")) qcCmdR2.output = new File(outDir, fastqR2Qc.get.getName.stripSuffix(".gz"))
qcCmdR2.read = "R2" qcCmdR2.read = "R2"
addSummarizable(qcCmdR2, "qc_command_R2") addSummarizable(qcCmdR2, "qc_command_R2")
...@@ -195,8 +197,8 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with ...@@ -195,8 +197,8 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
fqSync.refFastq = R1_in fqSync.refFastq = R1_in
fqSync.inputFastq1 = qcCmdR1.output fqSync.inputFastq1 = qcCmdR1.output
fqSync.inputFastq2 = qcCmdR2.output fqSync.inputFastq2 = qcCmdR2.output
fqSync.outputFastq1 = fastqR1Qc fqSync.outputFastq1 = new File(outDir, fastqR1Qc.getName)
fqSync.outputFastq2 = fastqR2Qc.get fqSync.outputFastq2 = new File(outDir, fastqR2Qc.get.getName)
fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats") fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats")
val pipe = new BiopetFifoPipe(this, fqSync :: Nil) { val pipe = new BiopetFifoPipe(this, fqSync :: Nil) {
...@@ -242,7 +244,7 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with ...@@ -242,7 +244,7 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
outputFiles += (chunk + "output_R1" -> R1) outputFiles += (chunk + "output_R1" -> R1)
if (paired) outputFiles += (chunk + "output_R2" -> R2.get) if (paired) outputFiles += (chunk + "output_R2" -> R2.get)
(R1, R2, deps) (R1, R2)
} }
/** Adds last non chunkable jobs */ /** Adds last non chunkable jobs */
......
...@@ -204,13 +204,11 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -204,13 +204,11 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
for ((chunkDir, fastqfile) <- chunks) { for ((chunkDir, fastqfile) <- chunks) {
var R1 = fastqfile._1 var R1 = fastqfile._1
var R2 = fastqfile._2 var R2 = fastqfile._2
var deps: List[File] = Nil
if (!skipFlexiprep) { if (!skipFlexiprep) {
val flexiout = flexiprep.runTrimClip(R1, R2, new File(chunkDir, "flexiprep"), chunkDir) val flexiout = flexiprep.runTrimClip(R1, R2, new File(chunkDir, "flexiprep"), chunkDir)
logger.debug(chunkDir + " - " + flexiout) logger.debug(chunkDir + " - " + flexiout)
R1 = flexiout._1 R1 = flexiout._1
if (paired) R2 = flexiout._2 if (paired) R2 = flexiout._2
deps = flexiout._3
fastq_R1_output :+= R1 fastq_R1_output :+= R1
R2.foreach(R2 => fastq_R2_output :+= R2) R2.foreach(R2 => fastq_R2_output :+= R2)
} }
...@@ -218,15 +216,15 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -218,15 +216,15 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
val outputBam = new File(chunkDir, outputName + ".bam") val outputBam = new File(chunkDir, outputName + ".bam")
bamFiles :+= outputBam bamFiles :+= outputBam
aligner match { aligner match {
case "bwa-mem" => addBwaMem(R1, R2, outputBam, deps) case "bwa-mem" => addBwaMem(R1, R2, outputBam)
case "bwa-aln" => addBwaAln(R1, R2, outputBam, deps) case "bwa-aln" => addBwaAln(R1, R2, outputBam)
case "bowtie" => addBowtie(R1, R2, outputBam, deps) case "bowtie" => addBowtie(R1, R2, outputBam)
case "gsnap" => addGsnap(R1, R2, outputBam, deps) case "gsnap" => addGsnap(R1, R2, outputBam)
// TODO: make TopHat here accept multiple input files // TODO: make TopHat here accept multiple input files
case "tophat" => addTophat(R1, R2, outputBam, deps) case "tophat" => addTophat(R1, R2, outputBam)
case "stampy" => addStampy(R1, R2, outputBam, deps) case "stampy" => addStampy(R1, R2, outputBam)
case "star" => addStar(R1, R2, outputBam, deps) case "star" => addStar(R1, R2, outputBam)
case "star-2pass" => addStar2pass(R1, R2, outputBam, deps) case "star-2pass" => addStar2pass(R1, R2, outputBam)
case _ => throw new IllegalStateException("Option aligner: '" + aligner + "' is not valid") case _ => throw new IllegalStateException("Option aligner: '" + aligner + "' is not valid")
} }
if (chunking && numberChunks.getOrElse(1) > 1 && config("chunk_metrics", default = false)) if (chunking && numberChunks.getOrElse(1) > 1 && config("chunk_metrics", default = false))
...@@ -267,10 +265,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -267,10 +265,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
} }
/** Add bwa aln jobs */ /** Add bwa aln jobs */
def addBwaAln(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addBwaAln(R1: File, R2: Option[File], output: File): File = {
val bwaAlnR1 = new BwaAln(this) val bwaAlnR1 = new BwaAln(this)
bwaAlnR1.fastq = R1 bwaAlnR1.fastq = R1
bwaAlnR1.deps = deps
bwaAlnR1.output = swapExt(output.getParent, output, ".bam", ".R1.sai") bwaAlnR1.output = swapExt(output.getParent, output, ".bam", ".R1.sai")
bwaAlnR1.isIntermediate = true bwaAlnR1.isIntermediate = true
add(bwaAlnR1) add(bwaAlnR1)
...@@ -278,7 +275,6 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -278,7 +275,6 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
val samFile: File = if (paired) { val samFile: File = if (paired) {
val bwaAlnR2 = new BwaAln(this) val bwaAlnR2 = new BwaAln(this)
bwaAlnR2.fastq = R2.get bwaAlnR2.fastq = R2.get
bwaAlnR2.deps = deps
bwaAlnR2.output = swapExt(output.getParent, output, ".bam", ".R2.sai") bwaAlnR2.output = swapExt(output.getParent, output, ".bam", ".R2.sai")
bwaAlnR2.isIntermediate = true bwaAlnR2.isIntermediate = true
add(bwaAlnR2) add(bwaAlnR2)
...@@ -313,11 +309,10 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -313,11 +309,10 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
} }
/** Adds bwa mem jobs */ /** Adds bwa mem jobs */
def addBwaMem(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addBwaMem(R1: File, R2: Option[File], output: File): File = {
val bwaCommand = new BwaMem(this) val bwaCommand = new BwaMem(this)
bwaCommand.R1 = R1 bwaCommand.R1 = R1
if (paired) bwaCommand.R2 = R2.get if (paired) bwaCommand.R2 = R2.get
bwaCommand.deps = deps
bwaCommand.R = Some(getReadGroupBwa) bwaCommand.R = Some(getReadGroupBwa)
val sortSam = new SortSam(this) val sortSam = new SortSam(this)
sortSam.output = output sortSam.output = output
...@@ -325,10 +320,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -325,10 +320,9 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
output output
} }
def addGsnap(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addGsnap(R1: File, R2: Option[File], output: File): File = {
val gsnapCommand = new Gsnap(this) val gsnapCommand = new Gsnap(this)
gsnapCommand.input = if (paired) List(R1, R2.get) else List(R1) gsnapCommand.input = if (paired) List(R1, R2.get) else List(R1)
gsnapCommand.deps = deps
gsnapCommand.output = swapExt(output.getParent, output, ".bam", ".sam") gsnapCommand.output = swapExt(output.getParent, output, ".bam", ".sam")
gsnapCommand.isIntermediate = true gsnapCommand.isIntermediate = true
add(gsnapCommand) add(gsnapCommand)
...@@ -341,13 +335,12 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -341,13 +335,12 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
addAddOrReplaceReadGroups(reorderSam.output, output) addAddOrReplaceReadGroups(reorderSam.output, output)
} }
def addTophat(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addTophat(R1: File, R2: Option[File], output: File): File = {
// TODO: merge mapped and unmapped BAM ~ also dealing with validation errors in the unmapped BAM // TODO: merge mapped and unmapped BAM ~ also dealing with validation errors in the unmapped BAM
val tophat = new Tophat(this) val tophat = new Tophat(this)
tophat.R1 = tophat.R1 :+ R1 tophat.R1 = tophat.R1 :+ R1
if (paired) tophat.R2 = tophat.R2 :+ R2.get if (paired) tophat.R2 = tophat.R2 :+ R2.get
tophat.output_dir = new File(outputDir, "tophat_out") tophat.output_dir = new File(outputDir, "tophat_out")
tophat.deps = deps
// always output BAM // always output BAM
tophat.no_convert_bam = false tophat.no_convert_bam = false
// and always keep input ordering // and always keep input ordering
...@@ -384,7 +377,7 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -384,7 +377,7 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
addAddOrReplaceReadGroups(reorderSam.output, output) addAddOrReplaceReadGroups(reorderSam.output, output)
} }
/** Adds stampy jobs */ /** Adds stampy jobs */
def addStampy(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addStampy(R1: File, R2: Option[File], output: File): File = {
var RG: String = "ID:" + readgroupId + "," var RG: String = "ID:" + readgroupId + ","
RG += "SM:" + sampleId.get + "," RG += "SM:" + sampleId.get + ","
...@@ -399,7 +392,6 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -399,7 +392,6 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
val stampyCmd = new Stampy(this) val stampyCmd = new Stampy(this)
stampyCmd.R1 = R1 stampyCmd.R1 = R1
if (paired) stampyCmd.R2 = R2.get if (paired) stampyCmd.R2 = R2.get
stampyCmd.deps = deps
stampyCmd.readgroup = RG stampyCmd.readgroup = RG
stampyCmd.sanger = true stampyCmd.sanger = true
stampyCmd.output = this.swapExt(output.getParent, output, ".bam", ".sam") stampyCmd.output = this.swapExt(output.getParent, output, ".bam", ".sam")
...@@ -412,11 +404,10 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -412,11 +404,10 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
} }
/** Adds bowtie jobs */ /** Adds bowtie jobs */
def addBowtie(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addBowtie(R1: File, R2: Option[File], output: File): File = {
val bowtie = new Bowtie(this) val bowtie = new Bowtie(this)
bowtie.R1 = R1 bowtie.R1 = R1
if (paired) bowtie.R2 = R2 if (paired) bowtie.R2 = R2
bowtie.deps = deps
bowtie.output = this.swapExt(output.getParent, output, ".bam", ".sam") bowtie.output = this.swapExt(output.getParent, output, ".bam", ".sam")
bowtie.isIntermediate = true bowtie.isIntermediate = true
add(bowtie) add(bowtie)
...@@ -424,15 +415,15 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S ...@@ -424,15 +415,15 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
} }
/** Adds Star jobs */ /** Adds Star jobs */
def addStar(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addStar(R1: File, R2: Option[File], output: File): File = {
val starCommand = Star(this, R1, R2, outputDir, isIntermediate = true, deps = deps) val starCommand = Star(this, R1, R2, outputDir, isIntermediate = true)
add(starCommand) add(starCommand)
addAddOrReplaceReadGroups(starCommand.outputSam, output) addAddOrReplaceReadGroups(starCommand.outputSam, output)
} }
/** Adds Start 2 pass jobs */ /** Adds Start 2 pass jobs */
def addStar2pass(R1: File, R2: Option[File], output: File, deps: List[File]): File = { def addStar2pass(R1: File, R2: Option[File], output: File): File = {
val starCommand = Star._2pass(this, R1, R2, outputDir, isIntermediate = true, deps = deps) val starCommand = Star._2pass(this, R1, R2, outputDir, isIntermediate = true)
addAll(starCommand._2) addAll(starCommand._2)
addAddOrReplaceReadGroups(starCommand._1, output) addAddOrReplaceReadGroups(starCommand._1, output)
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment