Merge branch 'feature-downloadsample_gears' into 'develop'

Feature downloadsample gears

This is a change on request for project 122

See merge request !415

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -69,10 +69,14 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
     outputDir = outputDir.getAbsoluteFile
     init()
     biopetScript()
+    logger.info("Biopet script done")
 
-    if (disableScatter) for (function <- functions) function match {
-      case f: ScatterGatherableFunction => f.scatterCount = 1
-      case _                            =>
+    if (disableScatter) {
+      logger.info("Disable scatters")
+      for (function <- functions) function match {
+        case f: ScatterGatherableFunction => f.scatterCount = 1
+        case _                            =>
+      }
     }
 
     this match {
@@ -81,6 +85,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
       case _ => reportClass.foreach(add(_))
     }
 
+    logger.info("Running pre commands")
     for (function <- functions) function match {
       case f: BiopetCommandLineFunction =>
         f.preProcessExecutable()
@@ -94,7 +99,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
       globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
     else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, output directory cannot be created")
 
-    inputFiles.foreach { i =>
+    logger.info("Checking input files")
+    inputFiles.par.foreach { i =>
       if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
       if (!i.file.canRead) Logging.addError(s"Input file can not be read: ${i.file}")
       if (!i.file.isAbsolute) Logging.addError(s"Input file should be an absolute path: ${i.file}")
@@ -111,6 +117,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
     if (logger.isDebugEnabled) WriteDependencies.writeDependencies(functions, new File(outputDir, s".log/${qSettings.runName}.deps.json"))
 
     Logging.checkErrors()
+    logger.info("Script complete without errors")
   }
 
   /** Get implemented from org.broadinstitute.gatk.queue.QScript */

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/qiime/PickClosedReferenceOtus.scala

@@ -31,8 +31,8 @@ class PickClosedReferenceOtus(val root: Configurable) extends BiopetCommandLineF
 
   var outputDir: File = null
 
-  override def defaultThreads = 3
-  override def defaultCoreMemory = 16.0
+  override def defaultThreads = 1
+  override def defaultCoreMemory = 20.0
   def versionCommand = executable + " --version"
   def versionRegex = """Version: (.*)""".r
 
@@ -51,7 +51,8 @@ class PickClosedReferenceOtus(val root: Configurable) extends BiopetCommandLineF
   var suppressTaxonomyAssignment: Boolean = config("suppress_taxonomy_assignment", default = false)
 
   def otuTable = new File(outputDir, "otu_table.biom")
-  def otuMap = new File(outputDir, "uclust_ref_picked_otus" + File.separator + "seqs_otus.txt")
+  def otuMap = new File(outputDir, "uclust_ref_picked_otus" + File.separator +
+    inputFasta.getName.stripSuffix(".fna").stripSuffix(".fasta").stripSuffix(".fa") + "_otus.txt")
 
   @Output
   private var outputFiles: List[File] = Nil

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/qiime/PickOpenReferenceOtus.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
+package nl.lumc.sasc.biopet.extensions.qiime
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Version }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by pjvan_thof on 12/4/15.
+ */
+class PickOpenReferenceOtus(val root: Configurable) extends BiopetCommandLineFunction with Version {
+  executable = config("exe", default = "pick_open_reference_otus.py")
+
+  @Input(required = true)
+  var inputFasta: File = _
+
+  var outputDir: File = null
+
+  override def defaultThreads = 1
+  override def defaultCoreMemory = 20.0
+  def versionCommand = executable + " --version"
+  def versionRegex = """Version: (.*)""".r
+
+  @Input(required = false)
+  var parameterFp: Option[File] = config("parameter_fp")
+
+  @Input(required = false)
+  var referenceFp: Option[File] = config("reference_fp")
+
+  @Input(required = false)
+  var taxonomyFp: Option[File] = config("taxonomy_fp")
+
+  var force: Boolean = config("force", default = false)
+  var printOnly: Boolean = config("print_only", default = false)
+  var suppressTaxonomyAssignment: Boolean = config("suppress_taxonomy_assignment", default = false)
+  var percentSubsample: Option[Double] = config("percent_subsample")
+  var prefilterPercentId: Option[Double] = config("prefilter_percent_id")
+  var suppressStep4: Boolean = config("suppress_step4", default = false)
+  var minOtuSize: Option[Int] = config("min_otu_size")
+  var suppressAlignAndTree: Boolean = config("suppress_taxonomy_assignment", default = false)
+
+  def otuTable = new File(outputDir, "otu_table_mc2_w_tax.biom")
+  def failedOtuTable = new File(outputDir, "otu_table_mc2_w_tax_no_pynast_failures.biom")
+  def otuMap = new File(outputDir, "final_otu_map.txt")
+
+  @Output
+  private var outputFiles: List[File] = Nil
+
+  override def beforeGraph(): Unit = {
+    super.beforeGraph()
+    jobOutputFile = new File(outputDir, ".std.out")
+    outputFiles ::= otuTable
+    outputFiles ::= failedOtuTable
+    outputFiles ::= otuMap
+  }
+
+  def cmdLine = executable + required("-f") +
+    required("-i", inputFasta) +
+    required("-o", outputDir) +
+    optional("--reference_fp", referenceFp) +
+    optional("--parameter_fp", parameterFp) +
+    optional("--taxonomy_fp", taxonomyFp) +
+    conditional(force, "--force") +
+    conditional(printOnly, "--print_only") +
+    conditional(suppressTaxonomyAssignment, "--suppress_taxonomy_assignment") +
+    (if (threads > 1) required("-a") + required("-O", threads) else "") +
+    optional("--percent_subsample", percentSubsample) +
+    optional("--prefilter_percent_id", prefilterPercentId) +
+    conditional(suppressStep4, "--suppress_step4") +
+    optional("-min_otu_size", minOtuSize) +
+    conditional(suppressAlignAndTree, "--suppress_align_and_tree")
+
+}

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/seqtk/SeqtkSample.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
+package nl.lumc.sasc.biopet.extensions.seqtk
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Wrapper for the seqtk sample subcommand.
+ * Written based on seqtk version 1.0-r63-dirty.
+ */
+class SeqtkSample(val root: Configurable) extends Seqtk {
+
+  /** input file */
+  @Input(doc = "Input file (FASTQ or FASTA)", required = true)
+  var input: File = _
+
+  /** output file */
+  @Output(doc = "Output file", required = true)
+  var output: File = _
+
+  var s: Option[Int] = config("seed")
+
+  var sample: Double = _
+
+  def cmdLine = required(executable) +
+    " sample " +
+    optional("-s", s) +
+    required(input) +
+    (if (sample > 1) required(sample.toInt) else required(sample)) +
+    (if (outputAsStsout) "" else " > " + required(output))
+
+}

biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/MergeOtuMaps.scala

@@ -31,9 +31,14 @@ class MergeOtuMaps(val root: Configurable) extends ToolCommandFunction {
   @Output(doc = "Output", shortName = "output", required = true)
   var output: File = _
 
+  var skipPrefix: List[String] = config("skip_prefix", default = Nil)
+
   override def defaultCoreMemory = 6.0
 
-  override def cmdLine = super.cmdLine + repeat("-I", input) + required("-o", output)
+  override def cmdLine = super.cmdLine +
+    repeat("-I", input) +
+    required("-o", output) +
+    repeat("-p", skipPrefix)
 
 }
 

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/MergeOtuMaps.scala

@@ -24,7 +24,9 @@ import scala.io.Source
  * Created by pjvan_thof on 12/18/15.
  */
 object MergeOtuMaps extends ToolCommand {
-  case class Args(inputFiles: List[File] = Nil, outputFile: File = null) extends AbstractArgs
+  case class Args(inputFiles: List[File] = Nil,
+                  outputFile: File = null,
+                  skipPrefix: List[String] = Nil) extends AbstractArgs
 
   class OptParser extends AbstractOptParser {
     opt[File]('I', "input") minOccurs 2 required () unbounded () valueName "<file>" action { (x, c) =>
@@ -33,6 +35,9 @@ object MergeOtuMaps extends ToolCommand {
     opt[File]('o', "output") required () unbounded () maxOccurs 1 valueName "<file>" action { (x, c) =>
       c.copy(outputFile = x)
     }
+    opt[String]('p', "skipPrefix") unbounded () valueName "<text>" action { (x, c) =>
+      c.copy(skipPrefix = x :: c.skipPrefix)
+    }
   }
 
   /**
@@ -40,23 +45,24 @@ object MergeOtuMaps extends ToolCommand {
    */
   def main(args: Array[String]): Unit = {
     val argsParser = new OptParser
-    val commandArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
+    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
 
-    var map: Map[Long, String] = Map()
+    var map: Map[String, String] = Map()
 
-    for (inputFile <- commandArgs.inputFiles) {
+    for (inputFile <- cmdArgs.inputFiles) {
       logger.info(s"Start reading $inputFile")
       val reader = Source.fromFile(inputFile)
       reader.getLines().foreach { line =>
         val values = line.split("\t", 2)
-        val key = values.head.toLong
-        map += key -> (line.stripPrefix(s"$key") + map.getOrElse(key, ""))
+        val key = values.head
+        if (!cmdArgs.skipPrefix.exists(key.startsWith))
+          map += key -> (line.stripPrefix(s"$key") + map.getOrElse(key, ""))
       }
       reader.close()
     }
 
-    logger.info(s"Start writing to ${commandArgs.outputFile}")
-    val writer = new PrintWriter(commandArgs.outputFile)
+    logger.info(s"Start writing to ${cmdArgs.outputFile}")
+    val writer = new PrintWriter(cmdArgs.outputFile)
     map.foreach { case (key, list) => writer.println(key + list) }
     writer.close()
   }

biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/Logging.scala

@@ -39,6 +39,7 @@ object Logging {
 
   def addError(error: String, debug: String = null): Unit = {
     val msg = error + (if (debug != null && logger.isDebugEnabled) "; " + debug else "")
+    logger.error(msg)
     errors.append(new Exception(msg))
   }
 

docs/pipelines/gears.md

@@ -10,6 +10,7 @@ Pipeline analysis components include:
  
  - [Kraken, DerrickWood](https://github.com/DerrickWood/kraken)
  - [Qiime closed reference](http://qiime.org)
+ - [Qiime open reference](http://qiime.org)
  - [Qiime rtax](http://qiime.org) (**Experimental**)
  - SeqCount (**Experimental**)
 
@@ -23,6 +24,7 @@ This pipeline is used to analyse a group of samples. This pipeline only accepts
 | --- | ---- | ------- | -------- |
 | gears_use_kraken | Boolean | true | Run fastq file with kraken |
 | gears_use_qiime_closed | Boolean | false | Run fastq files with qiime with the closed reference module |
+| gears_use_qiime_open | Boolean | false | Run fastq files with qiime with the open reference module |
 | gears_use_qiime_rtax | Boolean | false |  Run fastq files with qiime with the rtax module |
 | gears_use_seq_count | Boolean | false | Produces raw count files |
 
@@ -75,6 +77,7 @@ Please refer [to our mapping pipeline](mapping.md) for information about how the
 | --- | ---- | ------- | -------- |
 | gears_use_kraken | Boolean | true | Run fastq file with kraken |
 | gears_use_qiime_closed | Boolean | false | Run fastq files with qiime with the closed reference module |
+| gears_use_qiime_open | Boolean | false | Run fastq files with qiime with the open reference module |
 | gears_use_qiime_rtax | Boolean | false |  Run fastq files with qiime with the rtax module |
 | gears_use_seq_count | Boolean | false | Produces raw count files |
 

flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Cutadapt.scala

@@ -29,7 +29,15 @@ import nl.lumc.sasc.biopet.utils.config.Configurable
 class Cutadapt(root: Configurable, fastqc: Fastqc) extends nl.lumc.sasc.biopet.extensions.Cutadapt(root) {
 
   val ignoreFastqcAdapters: Boolean = config("ignore_fastqc_adapters", default = false)
-  val customAdaptersConfig: Map[String, Any] = config("custom_adapters", default = Map.empty)
+
+  val customAdaptersEnd: Map[String, Any] = config("custom_adapters_end", default = Map())
+  adapter ++= customAdaptersEnd.values.map(_.toString)
+
+  val customAdaptersFront: Map[String, Any] = config("custom_adapters_front", default = Map())
+  front ++= customAdaptersFront.values.map(_.toString)
+
+  val customAdaptersAny: Map[String, Any] = config("custom_adapters_any", default = Map())
+  anywhere ++= customAdaptersAny.values.map(_.toString)
 
   /** Clipped adapter names from FastQC */
   protected def seqToName: Map[String, String] = {
@@ -42,7 +50,7 @@ class Cutadapt(root: Configurable, fastqc: Fastqc) extends nl.lumc.sasc.biopet.e
   }
 
   def customAdapters: Set[AdapterSequence] = {
-    customAdaptersConfig.flatMap(adapter => {
+    (customAdaptersEnd ++ customAdaptersFront ++ customAdaptersAny).flatMap(adapter => {
       adapter match {
         case (adapterName: String, sequence: String) =>
           Some(AdapterSequence(adapterName, sequence))

flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/QcCommand.scala

@@ -103,13 +103,11 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
     clip = if (!flexiprep.skipClip) {
       val cutadapt = clip.getOrElse(new Cutadapt(root, fastqc))
 
-      val foundAdapters = if (!cutadapt.ignoreFastqcAdapters) {
-        fastqc.foundAdapters.map(_.seq) ++ cutadapt.customAdapters.map(_.seq)
-      } else {
-        cutadapt.customAdapters.map(_.seq)
-      }
+      val foundAdapters: Set[String] = if (!cutadapt.ignoreFastqcAdapters) {
+        fastqc.foundAdapters.map(_.seq)
+      } else Set()
 
-      if (foundAdapters.nonEmpty) {
+      if (foundAdapters.nonEmpty || cutadapt.adapter.nonEmpty || cutadapt.front.nonEmpty || cutadapt.anywhere.nonEmpty) {
         cutadapt.fastqInput = seqtk.output
         cutadapt.fastqOutput = new File(output.getParentFile, input.getName + ".cutadapt.fq")
         cutadapt.statsOutput = new File(flexiprep.outputDir, s"${flexiprep.sampleId.getOrElse("x")}-${flexiprep.libId.getOrElse("x")}.$read.clip.stats")

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/Gears.scala

@@ -15,10 +15,11 @@
 package nl.lumc.sasc.biopet.pipelines.gears
 
 import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
-import nl.lumc.sasc.biopet.core.{ PipelineCommand, MultiSampleQScript }
+import nl.lumc.sasc.biopet.core.{ MultiSampleQScript, PipelineCommand }
 import nl.lumc.sasc.biopet.extensions.tools.MergeOtuMaps
-import nl.lumc.sasc.biopet.extensions.{ Gzip, Zcat, Ln }
+import nl.lumc.sasc.biopet.extensions.{ Gzip, Ln, Zcat }
 import nl.lumc.sasc.biopet.extensions.qiime.MergeOtuTables
+import nl.lumc.sasc.biopet.extensions.seqtk.SeqtkSample
 import nl.lumc.sasc.biopet.pipelines.flexiprep.Flexiprep
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.queue.QScript
@@ -36,6 +37,8 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
     Some(gearsReport)
   }
 
+  override def defaults = Map("mergeotumaps" -> Map("skip_prefix" -> "New."))
+
   override def fixedValues = Map("gearssingle" -> Map("skip_flexiprep" -> true))
 
   /** Init for pipeline */
@@ -52,22 +55,30 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
   }
 
   def qiimeClosedDir: Option[File] = {
-    if (samples.values.flatMap(_.gs.qiimeClosed).nonEmpty) {
+    if (samples.values.flatMap(_.gearsSingle.qiimeClosed).nonEmpty) {
       Some(new File(outputDir, "qiime_closed_reference"))
     } else None
+  }
 
+  def qiimeOpenDir: Option[File] = {
+    if (samples.values.flatMap(_.gearsSingle.qiimeOpen).nonEmpty) {
+      Some(new File(outputDir, "qiime_open_reference"))
+    } else None
   }
 
   def qiimeClosedOtuTable: Option[File] = qiimeClosedDir.map(new File(_, "otu_table.biom"))
   def qiimeClosedOtuMap: Option[File] = qiimeClosedDir.map(new File(_, "otu_map.txt"))
 
+  def qiimeOpenOtuTable: Option[File] = qiimeOpenDir.map(new File(_, "otu_table.biom"))
+  def qiimeOpenOtuMap: Option[File] = qiimeOpenDir.map(new File(_, "otu_map.txt"))
+
   /**
    * Method where the multisample jobs should be added, this will be executed only when running the -sample argument is not given.
    */
   def addMultiSampleJobs(): Unit = {
-    val gss = samples.values.flatMap(_.gs.qiimeClosed).toList
-    val closedOtuTables = gss.map(_.otuTable)
-    val closedOtuMaps = gss.map(_.otuMap)
+    val qiimeCloseds = samples.values.flatMap(_.gearsSingle.qiimeClosed).toList
+    val closedOtuTables = qiimeCloseds.map(_.otuTable)
+    val closedOtuMaps = qiimeCloseds.map(_.otuMap)
     require(closedOtuTables.size == closedOtuMaps.size)
     if (closedOtuTables.nonEmpty) {
       if (closedOtuTables.size > 1) {
@@ -85,14 +96,35 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
         add(Ln(qscript, closedOtuMaps.head, qiimeClosedOtuMap.get))
         add(Ln(qscript, closedOtuTables.head, qiimeClosedOtuTable.get))
       }
+    }
+
+    val qiimeOpens = samples.values.flatMap(_.gearsSingle.qiimeOpen).toList
+    val openOtuTables = qiimeOpens.map(_.otuTable)
+    val openOtuMaps = qiimeOpens.map(_.otuMap)
+    require(openOtuTables.size == openOtuMaps.size)
+    if (openOtuTables.nonEmpty) {
+      if (openOtuTables.size > 1) {
+        val mergeTables = new MergeOtuTables(qscript)
+        mergeTables.input = openOtuTables
+        mergeTables.outputFile = qiimeOpenOtuTable.get
+        add(mergeTables)
+
+        val mergeMaps = new MergeOtuMaps(qscript)
+        mergeMaps.input = openOtuMaps
+        mergeMaps.output = qiimeOpenOtuMap.get
+        add(mergeMaps)
 
-      //TODO: Plots
+      } else {
+        add(Ln(qscript, openOtuMaps.head, qiimeOpenOtuMap.get))
+        add(Ln(qscript, openOtuTables.head, qiimeOpenOtuTable.get))
+      }
 
     }
   }
 
   /**
    * Factory method for Sample class
+   *
    * @param id SampleId
    * @return Sample class
    */
@@ -101,6 +133,7 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
   class Sample(sampleId: String) extends AbstractSample(sampleId) {
     /**
      * Factory method for Library class
+     *
      * @param id SampleId
      * @return Sample class
      */
@@ -115,10 +148,9 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
       flexiprep.inputR2 = config("R2")
       flexiprep.outputDir = new File(libDir, "flexiprep")
 
-      lazy val gs = new GearsSingle(qscript)
-      gs.sampleId = Some(sampleId)
-      gs.libId = Some(libId)
-      gs.outputDir = libDir
+      val libraryGears: Boolean = config("library_gears", default = false)
+
+      lazy val gearsSingle = if (libraryGears) Some(new GearsSingle(qscript)) else None
 
       /** Function that add library jobs */
       protected def addJobs(): Unit = {
@@ -126,9 +158,15 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
         flexiprep.inputR2.foreach(inputFiles :+= InputFile(_, config("R2_md5")))
         add(flexiprep)
 
-        gs.fastqR1 = Some(flexiprep.fastqR1Qc)
-        gs.fastqR2 = flexiprep.fastqR2Qc
-        add(gs)
+        gearsSingle.foreach { gs =>
+          gs.sampleId = Some(sampleId)
+          gs.libId = Some(libId)
+          gs.outputDir = libDir
+
+          gs.fastqR1 = Some(addDownsample(flexiprep.fastqR1Qc, gs.outputDir))
+          gs.fastqR2 = flexiprep.fastqR2Qc.map(addDownsample(_, gs.outputDir))
+          add(gs)
+        }
       }
 
       /** Must return files to store into summary */
@@ -138,9 +176,9 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
       def summaryStats = Map()
     }
 
-    lazy val gs = new GearsSingle(qscript)
-    gs.sampleId = Some(sampleId)
-    gs.outputDir = sampleDir
+    lazy val gearsSingle = new GearsSingle(qscript)
+    gearsSingle.sampleId = Some(sampleId)
+    gearsSingle.outputDir = sampleDir
 
     /** Function to add sample jobs */
     protected def addJobs(): Unit = {
@@ -156,9 +194,9 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
         add(Zcat(qscript, flexipreps.flatMap(_.fastqR2Qc)) | new Gzip(qscript) > file)
       }
 
-      gs.fastqR1 = Some(mergeR1)
-      gs.fastqR2 = mergeR2
-      add(gs)
+      gearsSingle.fastqR1 = Some(addDownsample(mergeR1, gearsSingle.outputDir))
+      gearsSingle.fastqR2 = mergeR2.map(addDownsample(_, gearsSingle.outputDir))
+      add(gearsSingle)
     }
 
     /** Must return files to store into summary */
@@ -168,11 +206,28 @@ class Gears(val root: Configurable) extends QScript with MultiSampleQScript { qs
     def summaryStats: Any = Map()
   }
 
+  val downSample: Option[Double] = config("gears_downsample")
+
+  def addDownsample(input: File, dir: File): File = {
+    downSample match {
+      case Some(x) =>
+        val output = new File(dir, input.getName + ".fq.gz")
+        val seqtk = new SeqtkSample(this)
+        seqtk.input = input
+        seqtk.sample = x
+        add(seqtk | new Gzip(this) > output)
+        output
+      case _ => input
+    }
+  }
+
   /** Must return a map with used settings for this pipeline */
-  def summarySettings: Map[String, Any] = Map()
+  def summarySettings: Map[String, Any] = Map("gears_downsample" -> downSample)
 
   /** File to put in the summary for thie pipeline */
   def summaryFiles: Map[String, File] = (
+    qiimeOpenOtuTable.map("qiime_open_otu_table" -> _) ++
+    qiimeOpenOtuMap.map("qiime_open_otu_map" -> _) ++
     qiimeClosedOtuTable.map("qiime_closed_otu_table" -> _) ++
     qiimeClosedOtuMap.map("qiime_closed_otu_map" -> _)
   ).toMap

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsQiimeClosed.scala

@@ -18,15 +18,15 @@ import java.io.{ File, PrintWriter }
 
 import nl.lumc.sasc.biopet.core.summary.SummaryQScript
 import nl.lumc.sasc.biopet.core.SampleLibraryTag
-import nl.lumc.sasc.biopet.extensions.Flash
 import nl.lumc.sasc.biopet.extensions.qiime._
+import nl.lumc.sasc.biopet.extensions.seqtk.SeqtkSample
 import nl.lumc.sasc.biopet.utils.ConfigUtils
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.queue.QScript
 
 import scala.collection.mutable
 import scala.collection.mutable.ListBuffer
-import scala.xml.{ PrettyPrinter, Elem }
+import scala.xml.{ Elem, PrettyPrinter }
 
 /**
  * Created by pjvan_thof on 12/4/15.
@@ -43,6 +43,7 @@ class GearsQiimeClosed(val root: Configurable) extends QScript with SummaryQScri
 
   def init() = {
     require(fastqInput != null)
+    require(sampleId.isDefined)
   }
 
   private var _otuMap: File = _
@@ -60,7 +61,7 @@ class GearsQiimeClosed(val root: Configurable) extends QScript with SummaryQScri
     add(splitLib)