Merge remote-tracking branch 'remotes/origin/develop' into fix-extractAlignedFastq

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetJavaCommandLineFunction.scala

@@ -24,6 +24,7 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
   javaGCHeapFreeLimit = config("java_gc_heap_freelimit", default = 10)
   javaGCTimeLimit = config("java_gc_timelimit", default = 50)
 
+  override def defaultResidentFactor: Double = 1.5
   override def defaultVmemFactor: Double = 2.0
 
   /** Constructs java opts, this adds scala threads */

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/CommandLineResources.scala

@@ -37,9 +37,10 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
   var vmem: Option[String] = config("vmem")
   def defaultCoreMemory: Double = 2.0
   def defaultVmemFactor: Double = 1.4
+  def defaultResidentFactor: Double = 1.2
   var vmemFactor: Double = config("vmem_factor", default = defaultVmemFactor)
 
-  var residentFactor: Double = config("resident_factor", default = 1.2)
+  var residentFactor: Double = config("resident_factor", default = defaultResidentFactor)
 
   private var _coreMemory: Double = 2.0
   def coreMemory = _coreMemory

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Bgzip.scala

@@ -30,7 +30,7 @@ class Bgzip(val root: Configurable) extends BiopetCommandLineFunction {
   var output: File = null
 
   var f: Boolean = config("f", default = false)
-  executable = config("exe", default = "bgzip")
+  executable = config("exe", default = "bgzip", freeVar = false)
 
   def cmdLine = required(executable) +
     conditional(f, "-f") +

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/VariantEffectPredictor.scala

@@ -275,12 +275,12 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
     else Map()
   }
 
-  protected val removeOnConflict = Set("Output_file", "Command_line_options", "Run_time", "Start_time", "End_time", "Input_file_(format)", "Novel_/_existing_variants")
-  protected val nonNumber = Set("VEP_version_(API)", "Cache/Database", "Species")
+  protected val removeOnConflict = Set("Output_file", "Run_time", "Start_time", "End_time", "Novel_/_existing_variants", "Input_file_(format)")
+  protected val nonNumber = Set("VEP_version_(API)", "Cache/Database", "Species", "Command_line_options")
 
   override def resolveSummaryConflict(v1: Any, v2: Any, key: String): Any = {
     if (removeOnConflict.contains(key)) None
-    else if (nonNumber.contains(key)) v1
+    else if (nonNumber.contains(key)) v2
     else {
       (v1, v2) match {
         case (x1: Int, x2: Int) => x1 + x2
@@ -301,9 +301,10 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
 
     (for ((header, headerIndex) <- headers) yield {
       val name = header.stripPrefix("[").stripSuffix("]")
-      name.replaceAll(" ", "_") -> (contents.drop(headerIndex + 1).takeWhile(!isHeader(_)).map { line =>
+      name.replaceAll(" ", "_") -> (contents.drop(headerIndex + 1).takeWhile(!isHeader(_)).flatMap { line =>
         val values = line.split("\t", 2)
-        values.head.replaceAll(" ", "_") -> tryToParseNumber(values.last).getOrElse(0)
+        if (values.last.isEmpty || values.last == "-") None
+        else Some(values.head.replaceAll(" ", "_") -> tryToParseNumber(values.last).getOrElse(values.last))
       }.toMap)
     }).toMap
   }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/Picard.scala

@@ -60,7 +60,7 @@ abstract class Picard extends BiopetJavaCommandLineFunction with Version {
   def versionRegex = """Version: (.*)""".r
   override def versionExitcode = List(0, 1)
 
-  override def defaultCoreMemory = 3.0
+  override def defaultCoreMemory = 4.0
 
   override def getVersion = {
     if (jarFile == null) Picard.getBiopetPicardVersion

biopet-extensions/src/test/resources/vep.failed.metrics

+[VEP run statistics]
+VEP version (API)	82 (82)
+Cache/Database	/usr/local/Genomes/H.Sapiens/GRCh37/annotation/vep/homo_sapiens_merged/81_GRCh37
+Species	homo_sapiens
+Command line options	-i /exports/sasc/pjvan_thof/toucan_test/output/chunk/chr17_GL000204_random-0-81310/chr17_GL000204_random-0-81310.vcf.gz -o /exports/sasc/pjvan_thof/toucan_test/output/chunk/chr17_GL000204_random-0-81310/chr17_GL000204_random-0-81310.vep.vcf -v --everything --stats_text --cache --vcf --allow_non_variant --merged --species homo_sapiens --assembly GRCh37 --dir /usr/local/Genomes/H.Sapiens/GRCh37/annotation/vep --fasta /usr/local/Genomes/H.Sapiens/GRCh37/annotation/vep/reference.fa --fork 2 --cache_version 81 --port 3337 --db_version 81 --failed 1
+Start time	2016-08-17 13:44:42
+End time	2016-08-17 13:44:42
+Run time	0 seconds
+Input file (format)	/exports/sasc/pjvan_thof/toucan_test/output/chunk/chr17_GL000204_random-0-81310/chr17_GL000204_random-0-81310.vcf.gz (VCF)
+Output file	/exports/sasc/pjvan_thof/toucan_test/output/chunk/chr17_GL000204_random-0-81310/chr17_GL000204_random-0-81310.vep.vcf [text]
+
+[General statistics]
+Lines of input read	413
+Variants processed	
+Variants remaining after filtering	
+Lines of output written	
+Novel / existing variants	-
+Overlapped genes	
+Overlapped transcripts	
+Overlapped regulatory features	-
+
+[Variant classes]
+
+[Consequences (most severe)]
+
+[Consequences (all)]
+
+[Coding consequences]
+
+[SIFT summary]
+
+[PolyPhen summary]
+
+[Variants by chromosome]
+
+[Position in protein]

biopet-extensions/src/test/scala/nl/lumc/sasc/biopet/extensions/VariantEffectPredictorTest.scala

@@ -17,6 +17,7 @@ package nl.lumc.sasc.biopet.extensions
 import java.io.File
 import java.nio.file.Paths
 
+import nl.lumc.sasc.biopet.utils.ConfigUtils
 import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.Test
@@ -46,4 +47,26 @@ class VariantEffectPredictorTest extends TestNGSuite with Matchers {
 
   }
 
+  @Test
+  def testFailedSummaryStats = {
+    val file = new File(Paths.get(getClass.getResource("/vep.failed.metrics").toURI).toString)
+
+    val vep = new VariantEffectPredictor(null)
+    val stats = vep.parseStatsFile(file)
+  }
+
+  @Test
+  def testMergeFailSuccess: Unit = {
+    val file1 = new File(Paths.get(getClass.getResource("/vep.metrics").toURI).toString)
+    val vep1 = new VariantEffectPredictor(null)
+    val stats1 = vep1.parseStatsFile(file1)
+
+    val file2 = new File(Paths.get(getClass.getResource("/vep.failed.metrics").toURI).toString)
+    val vep2 = new VariantEffectPredictor(null)
+    val stats2 = vep2.parseStatsFile(file2)
+
+    ConfigUtils.mergeMaps(stats1, stats2, vep1.resolveSummaryConflict)
+    ConfigUtils.mergeMaps(stats2, stats1, vep1.resolveSummaryConflict)
+  }
+
 }

carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala

@@ -46,7 +46,7 @@ class Carp(val root: Configurable) extends QScript with MultisampleMappingTrait
     "samtoolsview" -> Map("q" -> 10)
   )
 
-  override def fixedValues = Map(
+  override def fixedValues = super.fixedValues ++ Map(
     "samtoolsview" -> Map(
       "h" -> true,
       "b" -> true

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMapping.scala

@@ -67,6 +67,8 @@ trait MultisampleMappingTrait extends MultiSampleQScript
 
   override def defaults = super.defaults ++ Map("reordersam" -> Map("allow_incomplete_dict_concordance" -> true))
 
+  override def fixedValues: Map[String, Any] = super.fixedValues ++ Map("gearssingle" -> Map("skip_flexiprep" -> true))
+
   /** In a default multisample mapping run there are no multsample jobs. This method can be overriden by other pipelines */
   def addMultiSampleJobs(): Unit = {
     // this code will be executed after all code of all samples is executed

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaReport.scala

@@ -32,7 +32,10 @@ class ShivaReport(val root: Configurable) extends ReportBuilderExtension {
 }
 
 /** Object for report generation for Shiva pipeline */
-object ShivaReport extends MultisampleMappingReportTrait {
+object ShivaReport extends ShivaReportTrait
+
+/** Trait for report generation for Shiva pipeline, this can be extended */
+trait ShivaReportTrait extends MultisampleMappingReportTrait {
 
   def variantcallingExecuted = summary.getValue("shiva", "settings", "multisample_variantcalling") match {
     case Some(true) => true

shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/variantcallers/VarscanCnsSingleSample.scala

@@ -60,7 +60,9 @@ class VarscanCnsSingleSample(val root: Configurable) extends Variantcaller {
       val varscan = new VarscanMpileup2cns(this)
       varscan.vcfSampleList = Some(sampleFile)
 
-      add(mpileup | new FixMpileup(this) | varscan | new Bgzip(this) > sampleVcf)
+      val variantcallingJob = mpileup | new FixMpileup(this) | varscan | new Bgzip(this) > sampleVcf
+      variantcallingJob.threadsCorrection = -2
+      add(variantcallingJob)
       add(Tabix(this, sampleVcf))
 
       sampleVcf