Merge branch 'develop' of git.lumc.nl:biopet/biopet into feature-gears

biopet-aggregate/copy-src.sh

+#!/bin/bash
+
+DIR=`readlink -f \`dirname $0\``
+
+cp -r $DIR/../*/*/src/* $DIR/src
+

biopet-aggregate/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>BiopetRoot</artifactId>
+        <groupId>nl.lumc.sasc</groupId>
+        <version>0.5.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>BiopetAggregate</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.8</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <version>1.9.5</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.10</artifactId>
+            <version>2.2.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>BiopetProtectedPackage</artifactId>
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>18.0</version>
+        </dependency>
+
+    </dependencies>
+
+</project>
\ No newline at end of file

biopet-aggregate/rm-src.sh

+#!/bin/bash
+
+DIR=`readlink -f \`dirname $0\``
+
+rm -r $DIR/src/main $DIR/src/test
+

biopet-aggregate/src/.gitignore

+main
+test

pom.xml

@@ -17,5 +17,6 @@
         <module>public</module>
         <module>protected</module>
         <module>external-example</module>
+        <!--<module>biopet-aggregate</module>-->
     </modules>
 </project>

protected/biopet-gatk-pipelines/src/main/scala/nl/lumc/sasc/biopet/pipelines/gatk/GatkVariantcalling.scala

@@ -153,11 +153,11 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
           scriptOutput.rawVcfFile = m2v.output
 
           val vcfFilter = new VcfFilter(this) {
-            override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
+            override def defaults = Map("min_sample_depth" -> 8,
               "min_alternate_depth" -> 2,
               "min_samples_pass" -> 1,
               "filter_ref_calls" -> true
-            ), super.defaults)
+            )
           }
           vcfFilter.inputVcf = m2v.output
           vcfFilter.outputVcf = swapExt(outputDir, m2v.output, ".vcf", ".filter.vcf.gz")

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala

@@ -39,31 +39,27 @@ class ShivaTest extends TestNGSuite with Matchers {
     val bool = Array(true, false)
 
     for (
-      s1 <- bool; s2 <- bool; s3 <- bool; multi <- bool; single <- bool;
-      library <- bool; dbsnp <- bool; covariates <- bool; realign <- bool; baseRecalibration <- bool
-    ) yield Array("", s1, s2, s3, multi, single, library, dbsnp, covariates, realign, baseRecalibration)
+      s1 <- bool; s2 <- bool; multi <- bool;
+      dbsnp <- bool; realign <- bool; baseRecalibration <- bool
+    ) yield Array("", s1, s2, multi, dbsnp, realign, baseRecalibration)
   }
 
   @Test(dataProvider = "shivaOptions")
-  def testShiva(f: String, sample1: Boolean, sample2: Boolean, sample3: Boolean,
-                multi: Boolean, single: Boolean, library: Boolean, dbsnp: Boolean,
-                covariates: Boolean, realign: Boolean, baseRecalibration: Boolean): Unit = {
+  def testShiva(f: String, sample1: Boolean, sample2: Boolean,
+                multi: Boolean, dbsnp: Boolean,
+                realign: Boolean, baseRecalibration: Boolean): Unit = {
     val map = {
       var m: Map[String, Any] = ShivaTest.config
       if (sample1) m = ConfigUtils.mergeMaps(ShivaTest.sample1, m)
       if (sample2) m = ConfigUtils.mergeMaps(ShivaTest.sample2, m)
-      if (sample3) m = ConfigUtils.mergeMaps(ShivaTest.sample3, m)
       if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp" -> "test"), m)
       ConfigUtils.mergeMaps(Map("multisample_variantcalling" -> multi,
-        "single_sample_variantcalling" -> single,
-        "library_variantcalling" -> library,
-        "use_analyze_covariates" -> covariates,
         "use_indel_realigner" -> realign,
         "use_base_recalibration" -> baseRecalibration), m)
 
     }
 
-    if (!sample1 && !sample2 && !sample3) { // When no samples
+    if (!sample1 && !sample2) { // When no samples
       intercept[IllegalArgumentException] {
         initPipeline(map).script()
       }
@@ -71,20 +67,18 @@ class ShivaTest extends TestNGSuite with Matchers {
       val pipeline = initPipeline(map)
       pipeline.script()
 
-      val numberLibs = (if (sample1) 1 else 0) + (if (sample2) 1 else 0) + (if (sample3) 2 else 0)
-      val numberSamples = (if (sample1) 1 else 0) + (if (sample2) 1 else 0) + (if (sample3) 1 else 0)
+      val numberLibs = (if (sample1) 1 else 0) + (if (sample2) 2 else 0)
+      val numberSamples = (if (sample1) 1 else 0) + (if (sample2) 1 else 0)
 
-      pipeline.functions.count(_.isInstanceOf[MarkDuplicates]) shouldBe (numberLibs + (if (sample3) 1 else 0))
+      pipeline.functions.count(_.isInstanceOf[MarkDuplicates]) shouldBe (numberLibs + (if (sample2) 1 else 0))
 
       // Gatk preprocess
-      pipeline.functions.count(_.isInstanceOf[IndelRealigner]) shouldBe (numberLibs + (if (sample3) 1 else 0)) * (if (realign) 1 else 0)
-      pipeline.functions.count(_.isInstanceOf[RealignerTargetCreator]) shouldBe (numberLibs + (if (sample3) 1 else 0)) * (if (realign) 1 else 0)
-      pipeline.functions.count(_.isInstanceOf[BaseRecalibrator]) shouldBe (if (dbsnp && baseRecalibration) numberLibs else 0) * (if (covariates) 2 else 1)
-      pipeline.functions.count(_.isInstanceOf[AnalyzeCovariates]) shouldBe (if (dbsnp && covariates && baseRecalibration) numberLibs else 0)
+      pipeline.functions.count(_.isInstanceOf[IndelRealigner]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
+      pipeline.functions.count(_.isInstanceOf[RealignerTargetCreator]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
+      pipeline.functions.count(_.isInstanceOf[BaseRecalibrator]) shouldBe (if (dbsnp && baseRecalibration) numberLibs else 0)
       pipeline.functions.count(_.isInstanceOf[PrintReads]) shouldBe (if (dbsnp && baseRecalibration) numberLibs else 0)
 
-      pipeline.functions.count(_.isInstanceOf[VcfStats]) shouldBe (if (multi) 2 else 0) +
-        (if (single) numberSamples * 2 else 0) + (if (library) numberLibs * 2 else 0)
+      pipeline.functions.count(_.isInstanceOf[VcfStats]) shouldBe (if (multi) 2 else 0)
     }
   }
 }
@@ -115,7 +109,6 @@ object ShivaTest {
     "dir" -> "test",
     "vep_script" -> "test",
     "output_dir" -> outputDir,
-    "reference" -> (outputDir + File.separator + "ref.fa"),
     "reference_fasta" -> (outputDir + File.separator + "ref.fa"),
     "gatk_jar" -> "test",
     "samtools" -> Map("exe" -> "test"),
@@ -147,23 +140,14 @@ object ShivaTest {
     )))
 
   val sample2 = Map(
-    "samples" -> Map("sample2" -> Map("libraries" -> Map(
+    "samples" -> Map("sample3" -> Map("libraries" -> Map(
       "lib1" -> Map(
         "R1" -> inputTouch("2_1_R1.fq"),
         "R2" -> inputTouch("2_1_R2.fq")
-      )
-    )
-    )))
-
-  val sample3 = Map(
-    "samples" -> Map("sample3" -> Map("libraries" -> Map(
-      "lib1" -> Map(
-        "R1" -> inputTouch("3_1_R1.fq"),
-        "R2" -> inputTouch("3_1_R2.fq")
       ),
       "lib2" -> Map(
-        "R1" -> inputTouch("3_2_R1.fq"),
-        "R2" -> inputTouch("3_2_R2.fq")
+        "R1" -> inputTouch("2_2_R1.fq"),
+        "R2" -> inputTouch("2_2_R2.fq")
       )
     )
     )))

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala

@@ -131,7 +131,6 @@ object ShivaVariantcallingTest {
     "cache" -> true,
     "dir" -> "test",
     "vep_script" -> "test",
-    "reference" -> (outputDir + File.separator + "ref.fa"),
     "reference_fasta" -> (outputDir + File.separator + "ref.fa"),
     "gatk_jar" -> "test",
     "samtools" -> Map("exe" -> "test"),

public/basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTrait.scala

@@ -35,10 +35,10 @@ trait BastyTrait extends MultiSampleQScript {
 
   def variantcallers = List("freebayes")
 
-  override def defaults = ConfigUtils.mergeMaps(Map(
+  override def defaults = Map(
     "ploidy" -> 1,
     "variantcallers" -> variantcallers
-  ), super.defaults)
+  )
 
   lazy val shiva: ShivaTrait = new Shiva(qscript)
 

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/CommandLineResources.scala

@@ -85,9 +85,11 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
     this.freeze()
   }
 
+  var threadsCorrection = 0
+
   protected def combineResources(commands: List[CommandLineResources]): Unit = {
     commands.foreach(_.setResources())
-    nCoresRequest = Some(commands.map(_.threads).sum)
+    nCoresRequest = Some(commands.map(_.threads).sum + threadsCorrection)
 
     _coreMemory = commands.map(cmd => cmd.coreMemeory * (cmd.threads.toDouble / threads.toDouble)).sum
     memoryLimit = Some(_coreMemory * threads)

public/biopet-extentsions/src/main/scala/nl/lumc/sasc/biopet/extensions/Bowtie.scala → public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Bowtie.scala

@@ -47,7 +47,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Refe
   override def defaultCoreMemory = 4.0
   override def defaultThreads = 8
 
-  var sam: Boolean = config("sam", default = true)
+  var sam: Boolean = config("sam", default = false)
   var sam_RG: Option[String] = config("sam-RG")
   var seedlen: Option[Int] = config("seedlen")
   var seedmms: Option[Int] = config("seedmms")