Merge pull request #164 from biopet/fix-BIOPET-752

Adding option to also run qiime with multiple samples at once

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/Gears.scala

@@ -17,10 +17,11 @@ package nl.lumc.sasc.biopet.pipelines.gears
 import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
 import nl.lumc.sasc.biopet.core.{MultiSampleQScript, PipelineCommand}
 import nl.lumc.sasc.biopet.extensions.tools.MergeOtuMaps
-import nl.lumc.sasc.biopet.extensions.{Gzip, Ln, Zcat}
-import nl.lumc.sasc.biopet.extensions.qiime.MergeOtuTables
+import nl.lumc.sasc.biopet.extensions.{Cat, Gzip, Ln, Zcat}
+import nl.lumc.sasc.biopet.extensions.qiime.{MergeOtuTables, PickOpenReferenceOtus}
 import nl.lumc.sasc.biopet.extensions.seqtk.SeqtkSample
 import nl.lumc.sasc.biopet.pipelines.flexiprep.Flexiprep
+import nl.lumc.sasc.biopet.utils.Logging
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.queue.QScript
 
@@ -41,12 +42,20 @@ class Gears(val parent: Configurable) extends QScript with MultiSampleQScript {
 
   override def fixedValues = Map("gearssingle" -> Map("skip_flexiprep" -> true))
 
+  val qiimeMultisampleOpenReference: Boolean =
+    config("qiime_multisample_open_reference", default = false)
+
   /** Init for pipeline */
   def init(): Unit = {}
 
   /** Pipeline itself */
   def biopetScript(): Unit = {
     addSamplesJobs()
+
+    if (qiimeMultisampleOpenReference && !samples.exists(_._2.gearsSingle.qiimeOpen.isDefined))
+      Logging.addError(
+        "You selected 'qiime_multisample_open_reference' but qiime_open_reference is not enabled")
+
     addSummaryJobs()
   }
 
@@ -116,6 +125,21 @@ class Gears(val parent: Configurable) extends QScript with MultiSampleQScript {
       }
 
     }
+
+    if (qiimeMultisampleOpenReference) {
+      val dir = new File(outputDir, "qiime_open_reference_multisample")
+
+      val cat = new Cat(this)
+      cat.input = samples.flatMap(_._2.gearsSingle.qiimeOpen).map(_.fastaInput).toList
+      cat.output = new File(dir, "combined.fna")
+      add(cat)
+
+      val openReference = new PickOpenReferenceOtus(this)
+      openReference.inputFasta = cat.output
+      openReference.outputDir = new File(dir, "pick_open_reference_otus")
+      add(openReference)
+
+    }
   }
 
   /**

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsQiimeClosed.scala

@@ -36,16 +36,10 @@ class GearsQiimeClosed(val parent: Configurable)
     with SummaryQScript
     with SampleLibraryTag {
 
-  var fastqInput: File = _
-
-  override def defaults = Map(
-    "splitlibrariesfastq" -> Map(
-      "barcode_type" -> "not-barcoded"
-    )
-  )
+  var fastaInput: File = _
 
   def init(): Unit = {
-    require(fastqInput != null)
+    require(fastaInput != null)
     require(sampleId.isDefined)
   }
 
@@ -57,17 +51,9 @@ class GearsQiimeClosed(val parent: Configurable)
 
   def biopetScript(): Unit = {
 
-    val splitLib = new SplitLibrariesFastq(this)
-    splitLib.input :+= fastqInput
-    splitLib.outputDir = new File(outputDir, "split_libraries_fastq")
-    sampleId.foreach(splitLib.sampleIds :+= _.replaceAll("_", "-"))
-    splitLib.isIntermediate = true
-    add(splitLib)
-
     val closedReference = new PickClosedReferenceOtus(this)
-    closedReference.inputFasta = addDownsample(
-      splitLib.outputSeqs,
-      new File(splitLib.outputDir, s"${sampleId.get}.downsample.fna"))
+    closedReference.inputFasta =
+      addDownsample(fastaInput, new File(outputDir, s"${sampleId.get}.downsample.fna"))
     closedReference.outputDir = new File(outputDir, "pick_closed_reference_otus")
     add(closedReference)
     _otuMap = closedReference.otuMap

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsQiimeOpen.scala

@@ -29,16 +29,10 @@ class GearsQiimeOpen(val parent: Configurable)
     with SummaryQScript
     with SampleLibraryTag {
 
-  var fastqInput: File = _
-
-  override def defaults = Map(
-    "splitlibrariesfastq" -> Map(
-      "barcode_type" -> "not-barcoded"
-    )
-  )
+  var fastaInput: File = _
 
   def init(): Unit = {
-    require(fastqInput != null)
+    require(fastaInput != null)
     require(sampleId.isDefined)
   }
 
@@ -50,17 +44,8 @@ class GearsQiimeOpen(val parent: Configurable)
 
   def biopetScript(): Unit = {
 
-    val splitLib = new SplitLibrariesFastq(this)
-    splitLib.input :+= fastqInput
-    splitLib.outputDir = new File(outputDir, "split_libraries_fastq")
-    sampleId.foreach(splitLib.sampleIds :+= _.replaceAll("_", "-"))
-    splitLib.isIntermediate = true
-    add(splitLib)
-
     val openReference = new PickOpenReferenceOtus(this)
-    openReference.inputFasta = addDownsample(
-      splitLib.outputSeqs,
-      new File(splitLib.outputDir, s"${sampleId.get}.downsample.fna"))
+    openReference.inputFasta = addDownsample(fastaInput, new File(outputDir, s"downsample.fna"))
     openReference.outputDir = new File(outputDir, "pick_open_reference_otus")
     add(openReference)
     _otuMap = openReference.otuMap

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsSingle.scala

@@ -17,6 +17,7 @@ package nl.lumc.sasc.biopet.pipelines.gears
 import nl.lumc.sasc.biopet.core.summary.SummaryQScript
 import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
 import nl.lumc.sasc.biopet.core.{PipelineCommand, SampleLibraryTag}
+import nl.lumc.sasc.biopet.extensions.qiime.SplitLibrariesFastq
 import nl.lumc.sasc.biopet.extensions.{Gzip, Zcat}
 import nl.lumc.sasc.biopet.pipelines.flexiprep.Flexiprep
 import nl.lumc.sasc.biopet.utils.Logging
@@ -51,6 +52,12 @@ class GearsSingle(val parent: Configurable)
   @Argument(required = false)
   var outputName: String = _
 
+  override def defaults = Map(
+    "splitlibrariesfastq" -> Map(
+      "barcode_type" -> "not-barcoded"
+    )
+  )
+
   def getOutputName: String = {
     if (outputName == null) {
       sampleId.getOrElse("noName") + libId.map("-" + _).getOrElse("")
@@ -221,6 +228,16 @@ class GearsSingle(val parent: Configurable)
       }
     }
 
+    lazy val splitLibrariesFastq: File = {
+      val splitLib = new SplitLibrariesFastq(this)
+      splitLib.input :+= combinedFastq
+      splitLib.outputDir = new File(outputDir, "split_libraries_fastq")
+      sampleId.foreach(splitLib.sampleIds :+= _.replaceAll("_", "-"))
+      splitLib.isIntermediate = true
+      add(splitLib)
+      splitLib.outputSeqs
+    }
+
     krakenScript foreach { kraken =>
       kraken.fastqR1 = mergedR1
       kraken.fastqR2 = mergedR2
@@ -242,14 +259,14 @@ class GearsSingle(val parent: Configurable)
 
     qiimeClosed foreach { qiimeClosed =>
       qiimeClosed.outputDir = new File(outputDir, "qiime_closed")
-      qiimeClosed.fastqInput = combinedFastq
+      qiimeClosed.fastaInput = splitLibrariesFastq
       add(qiimeClosed)
       outputFiles += "qiime_closed_otu_table" -> qiimeClosed.otuTable
     }
 
     qiimeOpen foreach { qiimeOpen =>
       qiimeOpen.outputDir = new File(outputDir, "qiime_open")
-      qiimeOpen.fastqInput = combinedFastq
+      qiimeOpen.fastaInput = splitLibrariesFastq
       add(qiimeOpen)
       outputFiles += "qiime_open_otu_table" -> qiimeOpen.otuTable
     }