Merge branch 'fix-gentrap_test_runtime' into 'develop'

Reducing number of tests in gentrap

Fixes #365 

See merge request !432

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala

@@ -27,7 +27,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
   @Argument(doc = "Only Sample", shortName = "s", required = false, fullName = "sample")
   private[core] val onlySamples: List[String] = Nil
 
-  require(globalConfig.map.contains("samples"), "No Samples found in config")
+  if (!globalConfig.map.contains("samples")) Logging.addError("No Samples found in config")
 
   /** Sample class with basic functions build in */
   abstract class AbstractSample(val sampleId: String) extends Summarizable { sample =>
@@ -190,7 +190,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
   val samples: Map[String, Sample] = sampleIds.map(id => id -> makeSample(id)).toMap
 
   /** Returns a list of all sampleIDs */
-  protected def sampleIds: Set[String] = ConfigUtils.any2map(globalConfig.map("samples")).keySet
+  protected def sampleIds: Set[String] = ConfigUtils.any2map(globalConfig.map.getOrElse("samples", Map())).keySet
 
   protected lazy val nameRegex = """^[a-zA-Z0-9][a-zA-Z0-9-_]+[a-zA-Z0-9]$""".r
   protected lazy val nameError = "has an invalid name. " +

carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala

@@ -21,7 +21,7 @@ import nl.lumc.sasc.biopet.utils.config.Config
 import nl.lumc.sasc.biopet.extensions.bwa.BwaMem
 import nl.lumc.sasc.biopet.extensions.macs2.Macs2CallPeak
 import nl.lumc.sasc.biopet.extensions.picard.{ MergeSamFiles, SortSam }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
 import org.apache.commons.io.FileUtils
 import org.broadinstitute.gatk.queue.QSettings
 import org.scalatest.Matchers
@@ -65,13 +65,15 @@ class CarpTest extends TestNGSuite with Matchers {
     }
 
     if (!sample1 && !sample2 && !sample3 && !threatment && !control) { // When no samples
-      intercept[IllegalArgumentException] {
+      intercept[IllegalStateException] {
         initPipeline(map).script()
       }
+      Logging.errors.clear()
     } else if (threatment && !control) { // If control of a samples does not exist in samples
       intercept[IllegalStateException] {
         initPipeline(map).script()
       }
+      Logging.errors.clear()
     } else { // When samples are correct
       val carp = initPipeline(map)
       carp.script()

gears/src/test/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsTest.scala

@@ -72,7 +72,7 @@ abstract class GearsTest extends TestNGSuite with Matchers {
     }
 
     if (!sample1 && !sample2) { // When no samples
-      intercept[IllegalArgumentException] {
+      intercept[IllegalStateException] {
         initPipeline(map).script()
       }
       Logging.errors.clear()

gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala

@@ -129,9 +129,6 @@ class Gentrap(val root: Configurable) extends QScript
   lazy val fragmentsPerGene = if (expMeasures().contains(ExpMeasures.FragmentsPerGene))
     Some(new FragmentsPerGene(this)) else None
 
-  lazy val fragmentsPerExon = if (expMeasures().contains(ExpMeasures.FragmentsPerExon))
-    Some(new FragmentsPerExon(this)) else None
-
   lazy val baseCounts = if (expMeasures().contains(ExpMeasures.BaseCounts))
     Some(new BaseCounts(this)) else None
 
@@ -144,7 +141,7 @@ class Gentrap(val root: Configurable) extends QScript
   lazy val cufflinksStrict = if (expMeasures().contains(ExpMeasures.CufflinksStrict))
     Some(new CufflinksStrict(this)) else None
 
-  def executedMeasures = (fragmentsPerGene :: fragmentsPerExon :: baseCounts :: cufflinksBlind ::
+  def executedMeasures = (fragmentsPerGene :: baseCounts :: cufflinksBlind ::
     cufflinksGuided :: cufflinksStrict :: Nil).flatten
 
   /** Whether to do simple variant calling on RNA or not */
@@ -253,7 +250,7 @@ object Gentrap extends PipelineCommand {
   }
 
   /** Converts string with underscores into camel-case strings */
-  private def camelize(ustring: String): String = ustring
+  private[gentrap] def camelize(ustring: String): String = ustring
     .split("_")
     .map(_.toLowerCase.capitalize)
     .mkString("")

gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala

@@ -22,14 +22,14 @@ import nl.lumc.sasc.biopet.extensions._
 import nl.lumc.sasc.biopet.extensions.gmap.Gsnap
 import nl.lumc.sasc.biopet.extensions.hisat.Hisat2
 import nl.lumc.sasc.biopet.extensions.tools.{ BaseCounter, WipeReads }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
 import nl.lumc.sasc.biopet.utils.config.Config
 import org.broadinstitute.gatk.queue.QSettings
 import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.{ DataProvider, Test }
 
-abstract class GentrapTestAbstract(val expressionMeasure: String, val aligner: Option[String]) extends TestNGSuite with Matchers {
+abstract class GentrapTestAbstract(val expressionMeasures: List[String]) extends TestNGSuite with Matchers {
 
   def initPipeline(map: Map[String, Any]): Gentrap = {
     new Gentrap() {
@@ -42,127 +42,127 @@ abstract class GentrapTestAbstract(val expressionMeasure: String, val aligner: O
     }
   }
 
-  /** Convenience method for making library config */
-  private def makeLibConfig(idx: Int, paired: Boolean = true) = {
-    val files = Map("R1" -> GentrapTest.inputTouch("test_R1.fq"))
-    if (paired) (s"lib_$idx", files ++ Map("R2" -> GentrapTest.inputTouch("test_R2.fq")))
-    else (s"lib_$idx", files)
-  }
-
-  /** Convenience type for sample config */
-  private type SamplesConfig = Map[String, Map[String, Map[String, Map[String, Map[String, String]]]]]
-
-  /** Convenience method for making a single sample config */
-  private def makeSampleConfig(sampleIdx: Int, numLibs: Int, paired: Boolean) =
-    (s"sample_$sampleIdx",
-      Map("libraries" ->
-        (1 to numLibs)
-        .map(n => makeLibConfig(n, paired))
-        .toMap
-      )
-    )
+  def strandProtocols = Array("non_specific", "dutp")
 
-  /** Convenience method for making all samples config */
-  private def makeSamplesConfig(numSamples: Int, numLibsEachSample: Int, pairMode: String): SamplesConfig =
-    Map("samples" ->
-      (1 to numSamples)
-      // if paired == "mixed", alternate paired/not paired between each number
-      .map(n => makeSampleConfig(n, numLibsEachSample, if (pairMode == "mixed") n % 2 == 0 else pairMode == "paired"))
-      .toMap
-    )
+  def aligner: Option[String] = None
+  def removeRiboReads: Option[Boolean] = Some(false)
+  def sample1: Boolean = true
+  def sample2: Boolean = true
+  def callVariants: Option[Boolean] = None
 
   @DataProvider(name = "expMeasuresstrandProtocol")
   def expMeasuresStrandProtocolProvider = {
-
-    //val sampleConfigs = Array(pairedOneSampleOneLib, pairedOneSampleTwoLib, pairedOneSampleThreeLib)
-    val sampleConfigs = for {
-      (sampleNum, libNum) <- Seq(
-        // check multiple libs for single run only ~ to trim down less-informative tests
-        // need to check 2 and 3 samples since multi-sample plotting differs when sample is 1 or 2 and 3
-        (1, 1), (1, 2), (2, 1), (3, 1)
-      )
-      libType <- Seq("paired", "single", "mixed")
-    } yield makeSamplesConfig(sampleNum, libNum, libType)
-
-    val strandProtocols = Array("non_specific", "dutp")
-
     for {
-      sampleConfig <- sampleConfigs.toArray
       strandProtocol <- strandProtocols
-      removeRiboReads <- Array(true, false)
-    } yield Array(sampleConfig, List(expressionMeasure), strandProtocol, removeRiboReads)
+    } yield Array(strandProtocol)
   }
 
   @Test(dataProvider = "expMeasuresstrandProtocol")
-  def testGentrap(sampleConfig: SamplesConfig, expMeasures: List[String], strandProtocol: String, removeRiboReads: Boolean) = {
+  def testGentrap(strandProtocol: String) = {
 
     val settings = Map(
       "output_dir" -> GentrapTest.outputDir,
       "gsnap" -> Map("db" -> "test", "dir" -> "test"),
-      "expression_measures" -> expMeasures,
-      "strand_protocol" -> strandProtocol,
-      "remove_ribosomal_reads" -> removeRiboReads
-    ) ++ aligner.map("aligner" -> _)
-    val config = ConfigUtils.mergeMaps(settings ++ sampleConfig, Map(GentrapTest.executables.toSeq: _*))
+      "expression_measures" -> expressionMeasures,
+      "strand_protocol" -> strandProtocol
+    ) ++
+      aligner.map("aligner" -> _) ++
+      removeRiboReads.map("remove_ribosomal_reads" -> _) ++
+      callVariants.map("call_variants" -> _)
+    val configs: List[Option[Map[String, Any]]] = List(Some(settings), (if (sample1) Some(GentrapTest.sample1) else None), (if (sample2) Some(GentrapTest.sample2) else None))
+    val config = configs.flatten.foldLeft(GentrapTest.executables)((a, b) => ConfigUtils.mergeMaps(a, b))
     val gentrap: Gentrap = initPipeline(config)
 
-    gentrap.script()
-    val functions = gentrap.functions.flatMap {
-      case f: BiopetFifoPipe => f.pipesJobs
-      case f: BiopetPipe     => f.pipesJobs
-      case f                 => List(f)
-    }.groupBy(_.getClass)
-    val numSamples = sampleConfig("samples").size
-
-    if (expMeasures.contains("fragments_per_gene"))
-      assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
-
-    if (expMeasures.contains("fragments_per_exon"))
-      assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
-
-    if (expMeasures.contains("base_counts"))
-      gentrap.functions.count(_.isInstanceOf[BaseCounter]) shouldBe numSamples
-
-    if (expMeasures.contains("cufflinks_strict")) {
-      assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
-      assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
-    }
-
-    if (expMeasures.contains("cufflinks_guided")) {
-      assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
-      assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
-    }
-
-    if (expMeasures.contains("cufflinks_blind")) {
-      assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
-      assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+    val numSamples = (sample1, sample2) match {
+      case (true, true) => 2
+      case (_, true)    => 1
+      case (true, _)    => 1
+      case _            => 0
     }
 
-    gentrap.removeRibosomalReads shouldBe removeRiboReads
-    gentrap.functions.exists(_.isInstanceOf[WipeReads]) shouldBe removeRiboReads
-
-    val classMap = Map(
-      "gsnap" -> classOf[Gsnap],
-      "tophat" -> classOf[Tophat],
-      "star" -> classOf[Star],
-      "star-2pass" -> classOf[Star],
-      "hisat2" -> classOf[Hisat2]
-    )
+    if (numSamples == 0) {
+      intercept[IllegalArgumentException] {
+        gentrap.script()
+      }
+      Logging.errors.clear()
+    } else {
+      gentrap.script()
+
+      val functions = gentrap.functions.flatMap {
+        case f: BiopetFifoPipe => f.pipesJobs
+        case f: BiopetPipe     => f.pipesJobs
+        case f                 => List(f)
+      }.groupBy(_.getClass)
+
+      gentrap.shivaVariantcalling.isDefined shouldBe callVariants.getOrElse(false)
+
+      gentrap.summarySettings.getOrElse("expression_measures", List()).asInstanceOf[List[String]].sorted shouldBe
+        expressionMeasures.map(Gentrap.camelize(_)).sorted
+      gentrap.summarySettings.get("call_variants") shouldBe Some(callVariants.getOrElse(false))
+      gentrap.summarySettings.get("remove_ribosomal_reads") shouldBe Some(removeRiboReads.getOrElse(false))
+      gentrap.summarySettings.get("strand_protocol") shouldBe Some(Gentrap.camelize(strandProtocol))
+
+      if (expressionMeasures.contains("fragments_per_gene"))
+        assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
+
+      if (expressionMeasures.contains("fragments_per_exon"))
+        assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
+
+      if (expressionMeasures.contains("base_counts"))
+        gentrap.functions.count(_.isInstanceOf[BaseCounter]) shouldBe numSamples
+
+      if (expressionMeasures.contains("cufflinks_strict")) {
+        assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+        assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+      }
+
+      if (expressionMeasures.contains("cufflinks_guided")) {
+        assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+        assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+      }
+
+      if (expressionMeasures.contains("cufflinks_blind")) {
+        assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+        assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+      }
+
+      gentrap.removeRibosomalReads shouldBe removeRiboReads.getOrElse(false)
+      gentrap.functions.exists(_.isInstanceOf[WipeReads]) shouldBe removeRiboReads.getOrElse(false)
+
+      val classMap = Map(
+        "gsnap" -> classOf[Gsnap],
+        "tophat" -> classOf[Tophat],
+        "star" -> classOf[Star],
+        "star-2pass" -> classOf[Star],
+        "hisat2" -> classOf[Hisat2]
+      )
 
-    val alignerClass = classMap.get(aligner.getOrElse("gsnap"))
+      val alignerClass = classMap.get(aligner.getOrElse("gsnap"))
 
-    alignerClass.foreach(c => assert(functions.keys.exists(_ == c)))
-    classMap.values.filterNot(Some(_) == alignerClass).foreach(x => assert(!functions.keys.exists(_ == x)))
+      alignerClass.foreach(c => assert(functions.keys.exists(_ == c)))
+      classMap.values.filterNot(Some(_) == alignerClass).foreach(x => assert(!functions.keys.exists(_ == x)))
+    }
   }
 
 }
 
-class GentrapFragmentsPerGeneTest extends GentrapTestAbstract("fragments_per_gene", None)
-//class GentrapFragmentsPerExonTest extends GentrapTestAbstract("fragments_per_exon", None)
-class GentrapBaseCountsTest extends GentrapTestAbstract("base_counts", None)
-class GentrapCufflinksStrictTest extends GentrapTestAbstract("cufflinks_strict", None)
-class GentrapCufflinksGuidedTest extends GentrapTestAbstract("cufflinks_guided", None)
-class GentrapCufflinksBlindTest extends GentrapTestAbstract("cufflinks_blind", None)
+class GentrapFragmentsPerGeneTest extends GentrapTestAbstract(List("fragments_per_gene"))
+//class GentrapFragmentsPerExonTest extends GentrapTestAbstract("fragments_per_exon")
+class GentrapBaseCountsTest extends GentrapTestAbstract(List("base_counts"))
+class GentrapCufflinksStrictTest extends GentrapTestAbstract(List("cufflinks_strict"))
+class GentrapCufflinksGuidedTest extends GentrapTestAbstract(List("cufflinks_guided"))
+class GentrapCufflinksBlindTest extends GentrapTestAbstract(List("cufflinks_blind"))
+class GentrapAllTest extends GentrapTestAbstract(List("fragments_per_gene", "base_counts", "cufflinks_strict", "cufflinks_guided", "cufflinks_blind"))
+class GentrapNoSamplesTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+  override def sample1 = false
+  override def sample2 = false
+}
+class GentrapRemoveRibosomeTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+  override def removeRiboReads = Some(true)
+}
+class GentrapCallVariantsTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+  override def callVariants = Some(true)
+}
 
 object GentrapTest {
   val outputDir = Files.createTempDir()
@@ -185,7 +185,7 @@ object GentrapTest {
   copyFile("ref.dict")
   copyFile("ref.fa.fai")
 
-  val executables = Map(
+  val executables: Map[String, Any] = Map(
     "reference_fasta" -> (outputDir + File.separator + "ref.fa"),
     "refFlat" -> (outputDir + File.separator + "ref.fa"),
     "annotation_gtf" -> (outputDir + File.separator + "ref.fa"),
@@ -193,7 +193,8 @@ object GentrapTest {
     "annotation_refflat" -> (outputDir + File.separator + "ref.fa"),
     "ribosome_refflat" -> (outputDir + File.separator + "ref.fa"),
     "varscan_jar" -> "test",
-    "rscript" -> Map("exe" -> "test")
+    "rscript" -> Map("exe" -> "test"),
+    "gatk_jar" -> "test"
   ) ++ Seq(
       // fastqc executables
       "fastqc", "seqtk", "sickle", "cutadapt",
@@ -204,4 +205,26 @@ object GentrapTest {
       // bam2wig executables
       "igvtools", "wigtobigwig"
     ).map { case exe => exe -> Map("exe" -> "test") }.toMap
+
+  val sample1: Map[String, Any] = Map(
+    "samples" -> Map("sample1" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
+      )
+    )
+    )))
+
+  val sample2: Map[String, Any] = Map(
+    "samples" -> Map("sample3" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
+      ),
+      "lib2" -> Map(
+        "R1" -> inputTouch("2_2_R1.fq"),
+        "R2" -> inputTouch("2_2_R2.fq")
+      )
+    )
+    )))
 }

mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingTest.scala

@@ -19,7 +19,7 @@ import java.io.{ File, FileOutputStream }
 import com.google.common.io.Files
 import nl.lumc.sasc.biopet.extensions.kraken.Kraken
 import nl.lumc.sasc.biopet.extensions.picard.{ MarkDuplicates, MergeSamFiles }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
 import nl.lumc.sasc.biopet.utils.config.Config
 import org.broadinstitute.gatk.queue.QSettings
 import org.scalatest.Matchers
@@ -72,9 +72,10 @@ trait MultisampleMappingTestTrait extends TestNGSuite with Matchers {
     }
 
     if (!sample1 && !sample2 && !sample3 && !sample4) { // When no samples
-      intercept[IllegalArgumentException] {
+      intercept[IllegalStateException] {
         initPipeline(map).script()
       }
+      Logging.errors.clear()
     } else if (sample4 && !bamToFastq && !correctReadgroups) {
       intercept[IllegalStateException] {
         initPipeline(map).script()

shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala

@@ -20,7 +20,7 @@ import com.google.common.io.Files
 import nl.lumc.sasc.biopet.extensions.gatk.{ BaseRecalibrator, IndelRealigner, PrintReads, RealignerTargetCreator }
 import nl.lumc.sasc.biopet.extensions.picard.MarkDuplicates
 import nl.lumc.sasc.biopet.extensions.tools.VcfStats
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
 import nl.lumc.sasc.biopet.utils.config.Config
 import org.broadinstitute.gatk.queue.QSettings
 import org.scalatest.Matchers
@@ -86,6 +86,7 @@ trait ShivaTestTrait extends TestNGSuite with Matchers {
       intercept[IllegalArgumentException] {
         initPipeline(map).script()
       }
+      Logging.errors.clear()
     } else {
       val pipeline = initPipeline(map)
       pipeline.script()