diff --git a/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala b/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala
index 68a68c51f528e8a3d6f1059cfbfb83438626c522..24c54632be200fb7b2c5c02eb7833f137bee6399 100644
--- a/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala
+++ b/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala
@@ -27,7 +27,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
@Argument(doc = "Only Sample", shortName = "s", required = false, fullName = "sample")
private[core] val onlySamples: List[String] = Nil
- require(globalConfig.map.contains("samples"), "No Samples found in config")
+ if (!globalConfig.map.contains("samples")) Logging.addError("No Samples found in config")
/** Sample class with basic functions build in */
abstract class AbstractSample(val sampleId: String) extends Summarizable { sample =>
@@ -190,7 +190,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
val samples: Map[String, Sample] = sampleIds.map(id => id -> makeSample(id)).toMap
/** Returns a list of all sampleIDs */
- protected def sampleIds: Set[String] = ConfigUtils.any2map(globalConfig.map("samples")).keySet
+ protected def sampleIds: Set[String] = ConfigUtils.any2map(globalConfig.map.getOrElse("samples", Map())).keySet
protected lazy val nameRegex = """^[a-zA-Z0-9][a-zA-Z0-9-_]+[a-zA-Z0-9]$""".r
protected lazy val nameError = "has an invalid name. " +
diff --git a/carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala b/carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala
index 5d8bbd8ad77129961813b0e328e8df27bce28b00..3ba85bccab141d308714e0dd1ec109262f2bc834 100644
--- a/carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala
+++ b/carp/src/test/scala/nl/lumc/sasc/biopet/pipelines/carp/CarpTest.scala
@@ -21,7 +21,7 @@ import nl.lumc.sasc.biopet.utils.config.Config
import nl.lumc.sasc.biopet.extensions.bwa.BwaMem
import nl.lumc.sasc.biopet.extensions.macs2.Macs2CallPeak
import nl.lumc.sasc.biopet.extensions.picard.{ MergeSamFiles, SortSam }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
import org.apache.commons.io.FileUtils
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
@@ -65,13 +65,15 @@ class CarpTest extends TestNGSuite with Matchers {
}
if (!sample1 && !sample2 && !sample3 && !threatment && !control) { // When no samples
- intercept[IllegalArgumentException] {
+ intercept[IllegalStateException] {
initPipeline(map).script()
}
+ Logging.errors.clear()
} else if (threatment && !control) { // If control of a samples does not exist in samples
intercept[IllegalStateException] {
initPipeline(map).script()
}
+ Logging.errors.clear()
} else { // When samples are correct
val carp = initPipeline(map)
carp.script()
diff --git a/gears/src/test/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsTest.scala b/gears/src/test/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsTest.scala
index c3c13da1bbe96505847a70625f014acdd4f7a1dd..01332f4c3f8e94a9b8d9b27d7b0246ba2fa5b731 100644
--- a/gears/src/test/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsTest.scala
+++ b/gears/src/test/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsTest.scala
@@ -72,7 +72,7 @@ abstract class GearsTest extends TestNGSuite with Matchers {
}
if (!sample1 && !sample2) { // When no samples
- intercept[IllegalArgumentException] {
+ intercept[IllegalStateException] {
initPipeline(map).script()
}
Logging.errors.clear()
diff --git a/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala b/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala
index ed2eb5aff3b54df7f9a77571049ae324ee141191..2582192a319ff36ded25fb359671d2d6a61112f9 100644
--- a/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala
+++ b/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala
@@ -129,9 +129,6 @@ class Gentrap(val root: Configurable) extends QScript
lazy val fragmentsPerGene = if (expMeasures().contains(ExpMeasures.FragmentsPerGene))
Some(new FragmentsPerGene(this)) else None
- lazy val fragmentsPerExon = if (expMeasures().contains(ExpMeasures.FragmentsPerExon))
- Some(new FragmentsPerExon(this)) else None
-
lazy val baseCounts = if (expMeasures().contains(ExpMeasures.BaseCounts))
Some(new BaseCounts(this)) else None
@@ -144,7 +141,7 @@ class Gentrap(val root: Configurable) extends QScript
lazy val cufflinksStrict = if (expMeasures().contains(ExpMeasures.CufflinksStrict))
Some(new CufflinksStrict(this)) else None
- def executedMeasures = (fragmentsPerGene :: fragmentsPerExon :: baseCounts :: cufflinksBlind ::
+ def executedMeasures = (fragmentsPerGene :: baseCounts :: cufflinksBlind ::
cufflinksGuided :: cufflinksStrict :: Nil).flatten
/** Whether to do simple variant calling on RNA or not */
@@ -253,7 +250,7 @@ object Gentrap extends PipelineCommand {
}
/** Converts string with underscores into camel-case strings */
- private def camelize(ustring: String): String = ustring
+ private[gentrap] def camelize(ustring: String): String = ustring
.split("_")
.map(_.toLowerCase.capitalize)
.mkString("")
diff --git a/gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala b/gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala
index 090c8099ec4c52b75fa7d00de8d83961823ce7bd..4f2ae6ea2121b6427a3086e28ba9e46085a8bf27 100644
--- a/gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala
+++ b/gentrap/src/test/scala/nl/lumc/sasc/biopet/pipelines/gentrap/GentrapTest.scala
@@ -22,14 +22,14 @@ import nl.lumc.sasc.biopet.extensions._
import nl.lumc.sasc.biopet.extensions.gmap.Gsnap
import nl.lumc.sasc.biopet.extensions.hisat.Hisat2
import nl.lumc.sasc.biopet.extensions.tools.{ BaseCounter, WipeReads }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
import nl.lumc.sasc.biopet.utils.config.Config
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.{ DataProvider, Test }
-abstract class GentrapTestAbstract(val expressionMeasure: String, val aligner: Option[String]) extends TestNGSuite with Matchers {
+abstract class GentrapTestAbstract(val expressionMeasures: List[String]) extends TestNGSuite with Matchers {
def initPipeline(map: Map[String, Any]): Gentrap = {
new Gentrap() {
@@ -42,127 +42,127 @@ abstract class GentrapTestAbstract(val expressionMeasure: String, val aligner: O
}
}
- /** Convenience method for making library config */
- private def makeLibConfig(idx: Int, paired: Boolean = true) = {
- val files = Map("R1" -> GentrapTest.inputTouch("test_R1.fq"))
- if (paired) (s"lib_$idx", files ++ Map("R2" -> GentrapTest.inputTouch("test_R2.fq")))
- else (s"lib_$idx", files)
- }
-
- /** Convenience type for sample config */
- private type SamplesConfig = Map[String, Map[String, Map[String, Map[String, Map[String, String]]]]]
-
- /** Convenience method for making a single sample config */
- private def makeSampleConfig(sampleIdx: Int, numLibs: Int, paired: Boolean) =
- (s"sample_$sampleIdx",
- Map("libraries" ->
- (1 to numLibs)
- .map(n => makeLibConfig(n, paired))
- .toMap
- )
- )
+ def strandProtocols = Array("non_specific", "dutp")
- /** Convenience method for making all samples config */
- private def makeSamplesConfig(numSamples: Int, numLibsEachSample: Int, pairMode: String): SamplesConfig =
- Map("samples" ->
- (1 to numSamples)
- // if paired == "mixed", alternate paired/not paired between each number
- .map(n => makeSampleConfig(n, numLibsEachSample, if (pairMode == "mixed") n % 2 == 0 else pairMode == "paired"))
- .toMap
- )
+ def aligner: Option[String] = None
+ def removeRiboReads: Option[Boolean] = Some(false)
+ def sample1: Boolean = true
+ def sample2: Boolean = true
+ def callVariants: Option[Boolean] = None
@DataProvider(name = "expMeasuresstrandProtocol")
def expMeasuresStrandProtocolProvider = {
-
- //val sampleConfigs = Array(pairedOneSampleOneLib, pairedOneSampleTwoLib, pairedOneSampleThreeLib)
- val sampleConfigs = for {
- (sampleNum, libNum) <- Seq(
- // check multiple libs for single run only ~ to trim down less-informative tests
- // need to check 2 and 3 samples since multi-sample plotting differs when sample is 1 or 2 and 3
- (1, 1), (1, 2), (2, 1), (3, 1)
- )
- libType <- Seq("paired", "single", "mixed")
- } yield makeSamplesConfig(sampleNum, libNum, libType)
-
- val strandProtocols = Array("non_specific", "dutp")
-
for {
- sampleConfig <- sampleConfigs.toArray
strandProtocol <- strandProtocols
- removeRiboReads <- Array(true, false)
- } yield Array(sampleConfig, List(expressionMeasure), strandProtocol, removeRiboReads)
+ } yield Array(strandProtocol)
}
@Test(dataProvider = "expMeasuresstrandProtocol")
- def testGentrap(sampleConfig: SamplesConfig, expMeasures: List[String], strandProtocol: String, removeRiboReads: Boolean) = {
+ def testGentrap(strandProtocol: String) = {
val settings = Map(
"output_dir" -> GentrapTest.outputDir,
"gsnap" -> Map("db" -> "test", "dir" -> "test"),
- "expression_measures" -> expMeasures,
- "strand_protocol" -> strandProtocol,
- "remove_ribosomal_reads" -> removeRiboReads
- ) ++ aligner.map("aligner" -> _)
- val config = ConfigUtils.mergeMaps(settings ++ sampleConfig, Map(GentrapTest.executables.toSeq: _*))
+ "expression_measures" -> expressionMeasures,
+ "strand_protocol" -> strandProtocol
+ ) ++
+ aligner.map("aligner" -> _) ++
+ removeRiboReads.map("remove_ribosomal_reads" -> _) ++
+ callVariants.map("call_variants" -> _)
+ val configs: List[Option[Map[String, Any]]] = List(Some(settings), (if (sample1) Some(GentrapTest.sample1) else None), (if (sample2) Some(GentrapTest.sample2) else None))
+ val config = configs.flatten.foldLeft(GentrapTest.executables)((a, b) => ConfigUtils.mergeMaps(a, b))
val gentrap: Gentrap = initPipeline(config)
- gentrap.script()
- val functions = gentrap.functions.flatMap {
- case f: BiopetFifoPipe => f.pipesJobs
- case f: BiopetPipe => f.pipesJobs
- case f => List(f)
- }.groupBy(_.getClass)
- val numSamples = sampleConfig("samples").size
-
- if (expMeasures.contains("fragments_per_gene"))
- assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
-
- if (expMeasures.contains("fragments_per_exon"))
- assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
-
- if (expMeasures.contains("base_counts"))
- gentrap.functions.count(_.isInstanceOf[BaseCounter]) shouldBe numSamples
-
- if (expMeasures.contains("cufflinks_strict")) {
- assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
- assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
- }
-
- if (expMeasures.contains("cufflinks_guided")) {
- assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
- assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
- }
-
- if (expMeasures.contains("cufflinks_blind")) {
- assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
- assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+ val numSamples = (sample1, sample2) match {
+ case (true, true) => 2
+ case (_, true) => 1
+ case (true, _) => 1
+ case _ => 0
}
- gentrap.removeRibosomalReads shouldBe removeRiboReads
- gentrap.functions.exists(_.isInstanceOf[WipeReads]) shouldBe removeRiboReads
-
- val classMap = Map(
- "gsnap" -> classOf[Gsnap],
- "tophat" -> classOf[Tophat],
- "star" -> classOf[Star],
- "star-2pass" -> classOf[Star],
- "hisat2" -> classOf[Hisat2]
- )
+ if (numSamples == 0) {
+ intercept[IllegalArgumentException] {
+ gentrap.script()
+ }
+ Logging.errors.clear()
+ } else {
+ gentrap.script()
+
+ val functions = gentrap.functions.flatMap {
+ case f: BiopetFifoPipe => f.pipesJobs
+ case f: BiopetPipe => f.pipesJobs
+ case f => List(f)
+ }.groupBy(_.getClass)
+
+ gentrap.shivaVariantcalling.isDefined shouldBe callVariants.getOrElse(false)
+
+ gentrap.summarySettings.getOrElse("expression_measures", List()).asInstanceOf[List[String]].sorted shouldBe
+ expressionMeasures.map(Gentrap.camelize(_)).sorted
+ gentrap.summarySettings.get("call_variants") shouldBe Some(callVariants.getOrElse(false))
+ gentrap.summarySettings.get("remove_ribosomal_reads") shouldBe Some(removeRiboReads.getOrElse(false))
+ gentrap.summarySettings.get("strand_protocol") shouldBe Some(Gentrap.camelize(strandProtocol))
+
+ if (expressionMeasures.contains("fragments_per_gene"))
+ assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
+
+ if (expressionMeasures.contains("fragments_per_exon"))
+ assert(gentrap.functions.exists(_.isInstanceOf[HtseqCount]))
+
+ if (expressionMeasures.contains("base_counts"))
+ gentrap.functions.count(_.isInstanceOf[BaseCounter]) shouldBe numSamples
+
+ if (expressionMeasures.contains("cufflinks_strict")) {
+ assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+ assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+ }
+
+ if (expressionMeasures.contains("cufflinks_guided")) {
+ assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+ assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+ }
+
+ if (expressionMeasures.contains("cufflinks_blind")) {
+ assert(gentrap.functions.exists(_.isInstanceOf[Cufflinks]))
+ assert(gentrap.functions.exists(_.isInstanceOf[Ln]))
+ }
+
+ gentrap.removeRibosomalReads shouldBe removeRiboReads.getOrElse(false)
+ gentrap.functions.exists(_.isInstanceOf[WipeReads]) shouldBe removeRiboReads.getOrElse(false)
+
+ val classMap = Map(
+ "gsnap" -> classOf[Gsnap],
+ "tophat" -> classOf[Tophat],
+ "star" -> classOf[Star],
+ "star-2pass" -> classOf[Star],
+ "hisat2" -> classOf[Hisat2]
+ )
- val alignerClass = classMap.get(aligner.getOrElse("gsnap"))
+ val alignerClass = classMap.get(aligner.getOrElse("gsnap"))
- alignerClass.foreach(c => assert(functions.keys.exists(_ == c)))
- classMap.values.filterNot(Some(_) == alignerClass).foreach(x => assert(!functions.keys.exists(_ == x)))
+ alignerClass.foreach(c => assert(functions.keys.exists(_ == c)))
+ classMap.values.filterNot(Some(_) == alignerClass).foreach(x => assert(!functions.keys.exists(_ == x)))
+ }
}
}
-class GentrapFragmentsPerGeneTest extends GentrapTestAbstract("fragments_per_gene", None)
-//class GentrapFragmentsPerExonTest extends GentrapTestAbstract("fragments_per_exon", None)
-class GentrapBaseCountsTest extends GentrapTestAbstract("base_counts", None)
-class GentrapCufflinksStrictTest extends GentrapTestAbstract("cufflinks_strict", None)
-class GentrapCufflinksGuidedTest extends GentrapTestAbstract("cufflinks_guided", None)
-class GentrapCufflinksBlindTest extends GentrapTestAbstract("cufflinks_blind", None)
+class GentrapFragmentsPerGeneTest extends GentrapTestAbstract(List("fragments_per_gene"))
+//class GentrapFragmentsPerExonTest extends GentrapTestAbstract("fragments_per_exon")
+class GentrapBaseCountsTest extends GentrapTestAbstract(List("base_counts"))
+class GentrapCufflinksStrictTest extends GentrapTestAbstract(List("cufflinks_strict"))
+class GentrapCufflinksGuidedTest extends GentrapTestAbstract(List("cufflinks_guided"))
+class GentrapCufflinksBlindTest extends GentrapTestAbstract(List("cufflinks_blind"))
+class GentrapAllTest extends GentrapTestAbstract(List("fragments_per_gene", "base_counts", "cufflinks_strict", "cufflinks_guided", "cufflinks_blind"))
+class GentrapNoSamplesTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+ override def sample1 = false
+ override def sample2 = false
+}
+class GentrapRemoveRibosomeTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+ override def removeRiboReads = Some(true)
+}
+class GentrapCallVariantsTest extends GentrapTestAbstract(List("fragments_per_gene")) {
+ override def callVariants = Some(true)
+}
object GentrapTest {
val outputDir = Files.createTempDir()
@@ -185,7 +185,7 @@ object GentrapTest {
copyFile("ref.dict")
copyFile("ref.fa.fai")
- val executables = Map(
+ val executables: Map[String, Any] = Map(
"reference_fasta" -> (outputDir + File.separator + "ref.fa"),
"refFlat" -> (outputDir + File.separator + "ref.fa"),
"annotation_gtf" -> (outputDir + File.separator + "ref.fa"),
@@ -193,7 +193,8 @@ object GentrapTest {
"annotation_refflat" -> (outputDir + File.separator + "ref.fa"),
"ribosome_refflat" -> (outputDir + File.separator + "ref.fa"),
"varscan_jar" -> "test",
- "rscript" -> Map("exe" -> "test")
+ "rscript" -> Map("exe" -> "test"),
+ "gatk_jar" -> "test"
) ++ Seq(
// fastqc executables
"fastqc", "seqtk", "sickle", "cutadapt",
@@ -204,4 +205,26 @@ object GentrapTest {
// bam2wig executables
"igvtools", "wigtobigwig"
).map { case exe => exe -> Map("exe" -> "test") }.toMap
+
+ val sample1: Map[String, Any] = Map(
+ "samples" -> Map("sample1" -> Map("libraries" -> Map(
+ "lib1" -> Map(
+ "R1" -> inputTouch("1_1_R1.fq"),
+ "R2" -> inputTouch("1_1_R2.fq")
+ )
+ )
+ )))
+
+ val sample2: Map[String, Any] = Map(
+ "samples" -> Map("sample3" -> Map("libraries" -> Map(
+ "lib1" -> Map(
+ "R1" -> inputTouch("2_1_R1.fq"),
+ "R2" -> inputTouch("2_1_R2.fq")
+ ),
+ "lib2" -> Map(
+ "R1" -> inputTouch("2_2_R1.fq"),
+ "R2" -> inputTouch("2_2_R2.fq")
+ )
+ )
+ )))
}
diff --git a/mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingTest.scala b/mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingTest.scala
index 525005fdde51037050ea079cff46b5313f3c0e15..f036640ebb8700232b0b6f057cc45f48b64e3a36 100644
--- a/mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingTest.scala
+++ b/mapping/src/test/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingTest.scala
@@ -19,7 +19,7 @@ import java.io.{ File, FileOutputStream }
import com.google.common.io.Files
import nl.lumc.sasc.biopet.extensions.kraken.Kraken
import nl.lumc.sasc.biopet.extensions.picard.{ MarkDuplicates, MergeSamFiles }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
import nl.lumc.sasc.biopet.utils.config.Config
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
@@ -72,9 +72,10 @@ trait MultisampleMappingTestTrait extends TestNGSuite with Matchers {
}
if (!sample1 && !sample2 && !sample3 && !sample4) { // When no samples
- intercept[IllegalArgumentException] {
+ intercept[IllegalStateException] {
initPipeline(map).script()
}
+ Logging.errors.clear()
} else if (sample4 && !bamToFastq && !correctReadgroups) {
intercept[IllegalStateException] {
initPipeline(map).script()
diff --git a/shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala b/shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala
index 56881f2dd5f2d48ef5cda84b87216106c57ce1ba..ef62700ca687b0153c21fa75510c867e06b7c0b4 100644
--- a/shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala
+++ b/shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala
@@ -20,7 +20,7 @@ import com.google.common.io.Files
import nl.lumc.sasc.biopet.extensions.gatk.{ BaseRecalibrator, IndelRealigner, PrintReads, RealignerTargetCreator }
import nl.lumc.sasc.biopet.extensions.picard.MarkDuplicates
import nl.lumc.sasc.biopet.extensions.tools.VcfStats
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
import nl.lumc.sasc.biopet.utils.config.Config
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
@@ -86,6 +86,7 @@ trait ShivaTestTrait extends TestNGSuite with Matchers {
intercept[IllegalArgumentException] {
initPipeline(map).script()
}
+ Logging.errors.clear()
} else {
val pipeline = initPipeline(map)
pipeline.script()