Merge branch 'develop' into patch-flexiprep-parse-adapters

protected/basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/Basty.scala

@@ -41,7 +41,7 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
     var outputSnps: FastaOutput = _
 
     protected def addJobs(): Unit = {
-      addLibsJobs()
+      addPerLibJobs()
       output = addGenerateFasta(sampleId, sampleDir)
       outputSnps = addGenerateFasta(sampleId, sampleDir, snpsOnly = true)
     }
@@ -56,11 +56,13 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
     gatkPipeline.biopetScript
     addAll(gatkPipeline.functions)
 
+    addSamplesJobs()
+  }
+
+  def addMultiSampleJobs(): Unit = {
     val refVariants = addGenerateFasta(null, outputDir + "reference/", outputName = "reference")
     val refVariantSnps = addGenerateFasta(null, outputDir + "reference/", outputName = "reference", snpsOnly = true)
 
-    addSamplesJobs()
-
     val catVariants = Cat(this, refVariants.variants :: samples.map(_._2.output.variants).toList, outputDir + "fastas/variant.fasta")
     add(catVariants)
     val catVariantsSnps = Cat(this, refVariantSnps.variants :: samples.map(_._2.outputSnps.variants).toList, outputDir + "fastas/variant.snps_only.fasta")
@@ -122,13 +124,14 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
 
       val gubbins = new RunGubbins(this)
       gubbins.fastafile = concensusVariants
-      gubbins.startingTree = raxmlBi.getBipartitionsFile
+      gubbins.startingTree = Some(raxmlBi.getBipartitionsFile)
       gubbins.outputDirectory = outputDir + dirSufixGubbins
       add(gubbins)
     }
 
     addTreeJobs(catVariantsSnps.output, catConsensusVariantsSnps.output, outputDir + "trees" + File.separator + "snps_only", "snps_only")
     addTreeJobs(catVariants.output, catConsensusVariants.output, outputDir + "trees" + File.separator + "snps_indels", "snps_indels")
+
   }
 
   def addGenerateFasta(sampleName: String, outputDir: String, outputName: String = null,

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/BaseRecalibrator.scala

@@ -12,7 +12,7 @@ class BaseRecalibrator(val root: Configurable) extends org.broadinstitute.gatk.q
   memoryLimit = Option(4)
   override val defaultVmem = "8G"
 
-  if (config.contains("scattercount")) scatterCount = config("scattercount")
+  if (config.contains("scattercount")) scatterCount = config("scattercount", default = 1)
   if (config.contains("dbsnp")) knownSites :+= new File(config("dbsnp").asString)
   if (config.contains("known_sites")) knownSites :+= new File(config("known_sites").asString)
 }

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/GatkGeneral.scala

@@ -19,7 +19,7 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction {
 
   if (config.contains("intervals")) intervals = config("intervals").asFileList
   if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
-  reference_sequence = config("reference")
-  gatk_key = config("gatk_key")
+  reference_sequence = config("reference", required = true)
+  if (config.contains("gatk_key")) gatk_key = config("gatk_key")
   if (config.contains("pedigree")) pedigree = config("pedigree").asFileList
 }

protected/biopet-gatk-pipelines/src/main/scala/nl/lumc/sasc/biopet/pipelines/gatk/GatkPipeline.scala

@@ -36,7 +36,6 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
 
   var singleSampleCalling = config("single_sample_calling", default = true)
   var reference: File = config("reference", required = true)
-  var dbsnp: File = config("dbsnp")
   var useAllelesOption: Boolean = config("use_alleles_option", default = false)
   val externalGvcfs = config("external_gvcfs_files", default = Nil).asFileList
 
@@ -72,7 +71,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
             samToFastq.isIntermediate = true
             qscript.add(samToFastq)
             mapping.input_R1 = samToFastq.fastqR1
-            mapping.input_R2 = samToFastq.fastqR2
+            mapping.input_R2 = Some(samToFastq.fastqR2)
             mapping.init
             mapping.biopetScript
             addAll(mapping.functions) // Add functions of mapping to curent function pool
@@ -127,7 +126,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
     gatkVariantcalling.outputDir = sampleDir + "/variantcalling/"
 
     protected def addJobs(): Unit = {
-      addLibsJobs()
+      addPerLibJobs()
       gatkVariantcalling.inputBams = libraries.map(_._2.mapping.finalBamFile).toList
       gatkVariantcalling.preProcesBams = false
       if (!singleSampleCalling) {
@@ -150,10 +149,11 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
     override def configPath: List[String] = super.configPath ::: "multisample" :: Nil
   }
 
-  def biopetScript() {
-    addSamplesJobs
+  def biopetScript(): Unit = {
+    addSamplesJobs()
+  }
 
-    //SampleWide jobs
+  def addMultiSampleJobs(): Unit = {
     val gvcfFiles: List[File] = if (mergeGvcfs && externalGvcfs.size + samples.size > 1) {
       val newFile = outputDir + "merged.gvcf.vcf.gz"
       add(CombineGVCFs(this, externalGvcfs ++ samples.map(_._2.gatkVariantcalling.scriptOutput.gvcfFile), newFile))

protected/biopet-gatk-pipelines/src/main/scala/nl/lumc/sasc/biopet/pipelines/gatk/GatkVariantcalling.scala

@@ -32,9 +32,6 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
   @Argument(doc = "Reference", shortName = "R", required = false)
   var reference: File = config("reference", required = true)
 
-  @Argument(doc = "Dbsnp", shortName = "dbsnp", required = false)
-  var dbsnp: File = config("dbsnp")
-
   @Argument(doc = "OutputName", required = false)
   var outputName: String = _
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetCommandLineFunctionTrait.scala

@@ -38,7 +38,7 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
   val defaultThreads = 1
 
   @Argument(doc = "Vmem", required = false)
-  var vmem: String = _
+  var vmem: Option[String] = None
   val defaultVmem: String = ""
 
   @Argument(doc = "Executable", required = false)
@@ -58,9 +58,9 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
     if (threads == 0) threads = getThreads(defaultThreads)
     if (threads > 1) nCoresRequest = Option(threads)
 
-    if (vmem == null) {
+    if (vmem.isEmpty) {
       vmem = config("vmem")
-      if (vmem == null && !defaultVmem.isEmpty) vmem = defaultVmem
+      if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
     }
     if (vmem != null) jobResourceRequests :+= "h_vmem=" + vmem
     jobName = configName + ":" + firstOutput.getName

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala

@@ -26,7 +26,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Argument }
  */
 trait MultiSampleQScript extends BiopetQScript {
   @Argument(doc = "Only Sample", shortName = "sample", required = false)
-  val onlySample: List[String] = Nil
+  private val onlySamples: List[String] = Nil
 
   require(Config.global.map.contains("samples"), "No Samples found in config")
 
@@ -94,7 +94,7 @@ trait MultiSampleQScript extends BiopetQScript {
     protected def addJobs()
 
     /** function add all libraries in one call */
-    protected final def addLibsJobs(): Unit = {
+    protected final def addPerLibJobs(): Unit = {
       for ((libraryId, library) <- libraries) {
         library.addAndTrackJobs()
       }
@@ -125,17 +125,24 @@ trait MultiSampleQScript extends BiopetQScript {
   val samples: Map[String, Sample] = sampleIds.map(id => id -> makeSample(id)).toMap
 
   /** Returns a list of all sampleIDs */
-  protected def sampleIds: Set[String] = if (onlySample != Nil) onlySample.toSet else {
-    ConfigUtils.any2map(Config.global.map("samples")).keySet
-  }
+  protected def sampleIds: Set[String] = ConfigUtils.any2map(Config.global.map("samples")).keySet
 
   /** Runs addAndTrackJobs method for each sample */
   final def addSamplesJobs() {
-    for ((sampleId, sample) <- samples) {
-      sample.addAndTrackJobs()
-    }
+    if (onlySamples.isEmpty) {
+      samples.foreach { case (sampleId, sample) => sample.addAndTrackJobs() }
+      addMultiSampleJobs()
+    } else onlySamples.foreach(sampleId => samples.get(sampleId) match {
+      case Some(sample) => sample.addAndTrackJobs()
+      case None         => logger.warn("sampleId '" + sampleId + "' not found")
+    })
   }
 
+  /**
+   * Method where the multisample jobs should be added, this will be executed only when running the -sample argument is not given
+   */
+  def addMultiSampleJobs()
+
   /** Stores sample state */
   private var currentSample: Option[String] = None
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Bowtie.scala

@@ -43,7 +43,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction {
   override val defaultThreads = 8
 
   var sam: Boolean = config("sam", default = true)
-  var sam_RG: String = config("sam-RG")
+  var sam_RG: Option[String] = config("sam-RG")
   var seedlen: Option[Int] = config("seedlen")
   var seedmms: Option[Int] = config("seedmms")
   var k: Option[Int] = config("k")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Cutadapt.scala

@@ -43,8 +43,8 @@ class Cutadapt(val root: Configurable) extends BiopetCommandLineFunction {
   if (config.contains("front")) for (adapter <- config("front").asList) opt_front += adapter.toString
 
   var opt_discard: Boolean = config("discard", default = false)
-  var opt_minimum_length: String = config("minimum_length", 1)
-  var opt_maximum_length: String = config("maximum_length")
+  var opt_minimum_length: Option[Int] = config("minimum_length", 1)
+  var opt_maximum_length: Option[Int] = config("maximum_length")
 
   def cmdLine = required(executable) +
     // options

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Raxml.scala

@@ -60,11 +60,11 @@ class Raxml(val root: Configurable) extends BiopetCommandLineFunction {
   private var out: List[File] = Nil
 
   var executableNonThreads: String = config("exe", default = "raxmlHPC")
-  var executableThreads: String = config("exe_pthreads")
+  var executableThreads: Option[String] = config("exe_pthreads")
 
   override def afterGraph {
     if (threads == 0) threads = getThreads(defaultThreads)
-    executable = if (threads > 1 && executableThreads != null) executableThreads else executableNonThreads
+    executable = if (threads > 1 && executableThreads.isDefined) executableThreads.get else executableNonThreads
     super.afterGraph
     out +:= getInfoFile
     f match {

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/RunGubbins.scala

@@ -24,7 +24,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Argument, Input, Output }
 class RunGubbins(val root: Configurable) extends BiopetCommandLineFunction {
 
   @Input(doc = "Contaminants", required = false)
-  var startingTree: File = config("starting_tree")
+  var startingTree: Option[File] = config("starting_tree")
 
   @Input(doc = "Fasta file", shortName = "FQ")
   var fastafile: File = _
@@ -36,21 +36,21 @@ class RunGubbins(val root: Configurable) extends BiopetCommandLineFunction {
   var outputDirectory: String = _
 
   executable = config("exe", default = "run_gubbins.py")
-  var outgroup: String = config("outgroup")
-  var filterPercentage: String = config("filter_percentage")
-  var treeBuilder: String = config("tree_builder")
+  var outgroup: Option[String] = config("outgroup")
+  var filterPercentage: Option[String] = config("filter_percentage")
+  var treeBuilder: Option[String] = config("tree_builder")
   var iterations: Option[Int] = config("iterations")
   var minSnps: Option[Int] = config("min_snps")
-  var convergeMethod: String = config("converge_method")
+  var convergeMethod: Option[String] = config("converge_method")
   var useTimeStamp: Boolean = config("use_time_stamp", default = false)
-  var prefix: String = config("prefix")
+  var prefix: Option[String] = config("prefix")
   var verbose: Boolean = config("verbose", default = false)
   var noCleanup: Boolean = config("no_cleanup", default = false)
 
   override def afterGraph: Unit = {
     super.afterGraph
     jobLocalDir = new File(outputDirectory)
-    if (prefix == null) prefix = fastafile.getName
+    if (prefix.isEmpty) prefix = Some(fastafile.getName)
     val out: List[String] = List(".recombination_predictions.embl",
       ".recombination_predictions.gff",
       ".branch_base_reconstruction.embl",

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Sickle.scala

@@ -43,7 +43,7 @@ class Sickle(val root: Configurable) extends BiopetCommandLineFunction {
   var fastqc: Fastqc = _
 
   executable = config("exe", default = "sickle", freeVar = false)
-  var qualityType: String = config("qualitytype")
+  var qualityType: Option[String] = config("qualitytype")
   var qualityThreshold: Option[Int] = config("qualityThreshold")
   var lengthThreshold: Option[Int] = config("lengthThreshold")
   var noFiveprime: Boolean = config("noFiveprime", default = false)
@@ -54,7 +54,7 @@ class Sickle(val root: Configurable) extends BiopetCommandLineFunction {
   override def versionCommand = executable + " --version"
 
   override def afterGraph {
-    if (qualityType == null && defaultQualityType != null) qualityType = defaultQualityType
+    if (qualityType.isEmpty) qualityType = Some(defaultQualityType)
   }
 
   def cmdLine = {

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/bwa/BwaMem.scala

@@ -34,7 +34,7 @@ class BwaMem(val root: Configurable) extends Bwa {
   @Output(doc = "Output file SAM", shortName = "output")
   var output: File = _
 
-  var R: String = config("R")
+  var R: Option[String] = config("R")
   var k: Option[Int] = config("k")
   var r: Option[Float] = config("r")
   var S: Boolean = config("S", default = false)
@@ -49,11 +49,11 @@ class BwaMem(val root: Configurable) extends Bwa {
   var e: Boolean = config("e", default = false)
   var A: Option[Int] = config("A")
   var B: Option[Int] = config("B")
-  var O: String = config("O")
-  var E: String = config("E")
-  var L: String = config("L")
+  var O: Option[String] = config("O")
+  var E: Option[String] = config("E")
+  var L: Option[String] = config("L")
   var U: Option[Int] = config("U")
-  var x: String = config("x")
+  var x: Option[String] = config("x")
   var p: Boolean = config("p", default = false)
   var v: Option[Int] = config("v")
   var T: Option[Int] = config("T")
@@ -61,7 +61,7 @@ class BwaMem(val root: Configurable) extends Bwa {
   var a: Boolean = config("a", default = false)
   var C: Boolean = config("C", default = false)
   var Y: Boolean = config("Y", default = false)
-  var I: String = config("I")
+  var I: Option[String] = config("I")
 
   override val defaultVmem = "6G"
   override val defaultThreads = 8

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/macs2/Macs2CallPeak.scala

@@ -30,12 +30,12 @@ class Macs2CallPeak(val root: Configurable) extends Macs2 {
   @Output(doc = "Output file gappedPeak")
   private var output_gapped: File = _
 
-  var fileformat: String = config("fileformat")
+  var fileformat: Option[String] = config("fileformat")
   var gsize: Option[Float] = config("gsize")
   var keepdup: Boolean = config("keep-dup", default = false)
   var buffersize: Option[Int] = config("buffer-size")
-  var outputdir: String = config("outputDir")
-  var name: String = config("name")
+  var outputdir: String = _
+  var name: Option[String] = config("name")
   var bdg: Boolean = config("bdg", default = false)
   var verbose: Boolean = config("verbose", default = false)
   var tsize: Option[Int] = config("tsize")
@@ -56,12 +56,14 @@ class Macs2CallPeak(val root: Configurable) extends Macs2 {
   var callsummits: Boolean = config("callsummits", default = false)
 
   override def afterGraph: Unit = {
-    output_narrow = new File(outputdir + name + ".narrowPeak")
-    output_broad = new File(outputdir + name + ".broadPeak")
-    output_xls = new File(outputdir + name + ".xls")
-    output_bdg = new File(outputdir + name + ".bdg")
-    output_r = new File(outputdir + name + ".r")
-    output_gapped = new File(outputdir + name + ".gappedPeak")
+    if (name.isEmpty) throw new IllegalArgumentException("Name is not defined")
+    if (outputdir == null) throw new IllegalArgumentException("Outputdir is not defined")
+    output_narrow = new File(outputdir + name.get + ".narrowPeak")
+    output_broad = new File(outputdir + name.get + ".broadPeak")
+    output_xls = new File(outputdir + name.get + ".xls")
+    output_bdg = new File(outputdir + name.get + ".bdg")
+    output_r = new File(outputdir + name.get + ".r")
+    output_gapped = new File(outputdir + name.get + ".gappedPeak")
   }
 
   def cmdLine = {

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/MarkDuplicates.scala

@@ -32,19 +32,19 @@ class MarkDuplicates(val root: Configurable) extends Picard {
   var outputMetrics: File = _
 
   @Argument(doc = "PROGRAM_RECORD_ID", required = false)
-  var programRecordId: String = config("programrecordid")
+  var programRecordId: Option[String] = config("programrecordid")
 
   @Argument(doc = "PROGRAM_GROUP_VERSION", required = false)
-  var programGroupVersion: String = config("programgroupversion")
+  var programGroupVersion: Option[String] = config("programgroupversion")
 
   @Argument(doc = "PROGRAM_GROUP_COMMAND_LINE", required = false)
-  var programGroupCommandLine: String = config("programgroupcommandline")
+  var programGroupCommandLine: Option[String] = config("programgroupcommandline")
 
   @Argument(doc = "PROGRAM_GROUP_NAME", required = false)
-  var programGroupName: String = config("programgroupname")
+  var programGroupName: Option[String] = config("programgroupname")
 
   @Argument(doc = "COMMENT", required = false)
-  var comment: String = config("comment")
+  var comment: Option[String] = config("comment")
 
   @Argument(doc = "REMOVE_DUPLICATES", required = false)
   var removeDuplicates: Boolean = config("removeduplicates", default = false)
@@ -62,7 +62,7 @@ class MarkDuplicates(val root: Configurable) extends Picard {
   var sortingCollectionSizeRatio: Option[Double] = config("sortingCollectionSizeRatio")
 
   @Argument(doc = "READ_NAME_REGEX", required = false)
-  var readNameRegex: String = config("readNameRegex")
+  var readNameRegex: Option[String] = config("readNameRegex")
 
   @Argument(doc = "OPTICAL_DUPLICATE_PIXEL_DISTANCE", required = false)
   var opticalDuplicatePixelDistance: Option[Int] = config("opticalDuplicatePixelDistance")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/MergeSamFiles.scala

@@ -41,7 +41,7 @@ class MergeSamFiles(val root: Configurable) extends Picard {
   var useThreading: Boolean = config("use_threading", default = false)
 
   @Argument(doc = "COMMENT", required = false)
-  var comment: String = config("comment")
+  var comment: Option[String] = config("comment")
 
   override def commandLine = super.commandLine +
     repeat("INPUT=", input, spaceSeparated = false) +

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/Picard.scala

@@ -22,13 +22,13 @@ abstract class Picard extends BiopetJavaCommandLineFunction {
   override def subPath = "picard" :: super.subPath
 
   @Argument(doc = "VERBOSITY", required = false)
-  var verbosity: String = config("verbosity")
+  var verbosity: Option[String] = config("verbosity")
 
   @Argument(doc = "QUIET", required = false)
   var quiet: Boolean = config("quiet", default = false)
 
   @Argument(doc = "VALIDATION_STRINGENCY", required = false)
-  var stringency: String = config("validationstringency")
+  var stringency: Option[String] = config("validationstringency")
 
   @Argument(doc = "COMPRESSION_LEVEL", required = false)
   var compression: Option[Int] = config("compressionlevel")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/SamToFastq.scala

@@ -38,7 +38,7 @@ class SamToFastq(val root: Configurable) extends Picard {
   var outputPerRg: Boolean = config("outputPerRg", default = false)
 
   @Argument(doc = "Output dir", required = false)
-  var outputDir: String = config("outputDir")
+  var outputDir: String = _
 
   @Argument(doc = "re reverse", required = false)
   var reReverse: Boolean = config("reReverse", default = false)
@@ -53,7 +53,7 @@ class SamToFastq(val root: Configurable) extends Picard {
   var clippingAtribute: String = config("clippingAtribute")
 
   @Argument(doc = "clippingAction", required = false)
-  var clippingAction: String = config("clippingAction")
+  var clippingAction: Option[String] = config("clippingAction")
 
   @Argument(doc = "read1Trim", required = false)
   var read1Trim: Option[Int] = config("read1Trim")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/samtools/SamtoolsMpileup.scala

@@ -30,7 +30,7 @@ class SamtoolsMpileup(val root: Configurable) extends Samtools {
   var reference: File = config("reference")
 
   @Input(doc = "Interval bed")
-  var intervalBed: File = config("interval_bed")
+  var intervalBed: Option[File] = config("interval_bed")
 
   var disableBaq: Boolean = config("disable_baq")
   var minMapQuality: Option[Int] = config("min_map_quality")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/seqtk/SeqtkSeq.scala

@@ -37,7 +37,7 @@ class SeqtkSeq(val root: Configurable) extends Seqtk {
   var q: Option[Int] = config("q")
 
   /** masked bases converted to CHAR; 0 for lowercase [0] */
-  var n: String = config("n")
+  var n: Option[String] = config("n")
 
   /** number of residues per line; 0 for 2^32-1 [0] */
   var l: Option[Int] = config("l")
@@ -52,7 +52,7 @@ class SeqtkSeq(val root: Configurable) extends Seqtk {
   var f: Option[Int] = config("f")
 
   /** mask regions in BED or name list FILE [null] */
-  var M: File = config("M")
+  var M: Option[File] = config("M")
 
   /** drop sequences with length shorter than INT [0] */
   var L: Option[Int] = config("L")