Merge branch 'feature-basty_testing' into 'develop'

Added testing to basty



See merge request !456

basty/pom.xml

@@ -54,5 +54,17 @@
             <artifactId>Mapping</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.8</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.10</artifactId>
+            <version>2.2.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/Basty.scala

@@ -33,10 +33,18 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
 
   def this() = this(null)
 
-  case class FastaOutput(variants: File, consensus: File, consensusVariants: File)
+  case class FastaOutput(variants: File, consensus: File, consensusVariants: File) {
+    def summaryFiles(prefix: Option[String] = None) = Map(
+      s"${prefix.map(_ + "_").getOrElse("")}variants_fasta" -> variants,
+      s"${prefix.map(_ + "_").getOrElse("")}consensus_fasta" -> consensus,
+      s"${prefix.map(_ + "_").getOrElse("")}consensus_variants_fasta" -> consensusVariants
+    )
+  }
 
   def variantcallers = List("unifiedgenotyper")
 
+  val numBoot = config("boot_runs", default = 100, namespace = "raxml").asInt
+
   override def defaults = Map(
     "ploidy" -> 1,
     "variantcallers" -> variantcallers
@@ -46,28 +54,26 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
 
   def summaryFile: File = new File(outputDir, "Basty.summary.json")
 
-  //TODO: Add summary
   def summaryFiles: Map[String, File] = Map()
 
-  //TODO: Add summary
-  def summarySettings: Map[String, Any] = Map()
+  def summarySettings: Map[String, Any] = Map("boot_runs" -> numBoot)
 
   def makeSample(id: String) = new Sample(id)
   class Sample(sampleId: String) extends AbstractSample(sampleId) {
-    //TODO: Add summary
-    def summaryFiles: Map[String, File] = Map()
+    def summaryFiles: Map[String, File] = output.summaryFiles() ++ outputSnps.summaryFiles(Some("snps_only"))
 
-    //TODO: Add summary
     def summaryStats: Map[String, Any] = Map()
 
+    override def summarySettings: Map[String, Any] = Map()
+
     def makeLibrary(id: String) = new Library(id)
     class Library(libId: String) extends AbstractLibrary(libId) {
-      //TODO: Add summary
       def summaryFiles: Map[String, File] = Map()
 
-      //TODO: Add summary
       def summaryStats: Map[String, Any] = Map()
 
+      override def summarySettings: Map[String, Any] = Map()
+
       protected def addJobs(): Unit = {}
     }
 
@@ -136,7 +142,6 @@ class Basty(val root: Configurable) extends QScript with MultiSampleQScript {
       add(raxmlMl)
 
       val r = new scala.util.Random(seed)
-      val numBoot = config("boot_runs", default = 100, namespace = "raxml").asInt
       val bootList = for (t <- 0 until numBoot) yield {
         val raxmlBoot = new Raxml(this)
         raxmlBoot.input = variants

basty/src/test/resources/ref.dict

+@HD	VN:1.4	SO:unsorted
+@SQ	SN:chr1	LN:9	UR:file:/home/pjvan_thof/pipelines/biopet/public/mapping/src/test/resources/ref.fa	M5:fe15dbbd0900310caf32827f6da57550

basty/src/test/resources/ref.fa

+>chr1
+AGTAGTAGT

basty/src/test/resources/ref.fa.fai

+chr1	9	6	9	10

basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala

+package nl.lumc.sasc.biopet.pipelines.basty
+
+import java.io.{ File, FileOutputStream }
+
+import com.google.common.io.Files
+import nl.lumc.sasc.biopet.extensions.{ Raxml, RunGubbins }
+import nl.lumc.sasc.biopet.extensions.gatk.{ BaseRecalibrator, IndelRealigner, PrintReads, RealignerTargetCreator }
+import nl.lumc.sasc.biopet.extensions.picard.MarkDuplicates
+import nl.lumc.sasc.biopet.extensions.tools.{ BastyGenerateFasta, VcfStats }
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
+import nl.lumc.sasc.biopet.utils.config.Config
+import org.broadinstitute.gatk.queue.QSettings
+import org.scalatest.Matchers
+import org.scalatest.testng.TestNGSuite
+import org.testng.annotations.{ DataProvider, Test }
+
+/**
+ * Created by pjvanthof on 27/09/16.
+ */
+class BastyTest extends TestNGSuite with Matchers {
+  def initPipeline(map: Map[String, Any]): Basty = {
+    new Basty() {
+      override def configNamespace = "shiva"
+      override def globalConfig = new Config(ConfigUtils.mergeMaps(map, BastyTest.config))
+      qSettings = new QSettings
+      qSettings.runName = "test"
+    }
+  }
+
+  @DataProvider(name = "bastyOptions")
+  def bastyOptions = {
+    for (
+      s1 <- sample1; s2 <- sample2
+    ) yield Array("", s1, s2)
+  }
+
+  def sample1 = Array(false, true)
+  def sample2 = Array(false, true)
+  def realign = true
+  def baseRecalibration = true
+  def multisampleCalling: Boolean = true
+  def sampleCalling = false
+  def libraryCalling = false
+  def dbsnp = false
+  def svCalling = false
+  def cnvCalling = false
+  def annotation = false
+  def bootRuns: Option[Int] = None
+
+  @Test(dataProvider = "bastyOptions")
+  def testBasty(f: String, sample1: Boolean, sample2: Boolean): Unit = {
+    val map = {
+      var m: Map[String, Any] = BastyTest.config
+      if (sample1) m = ConfigUtils.mergeMaps(BastyTest.sample1, m)
+      if (sample2) m = ConfigUtils.mergeMaps(BastyTest.sample2, m)
+      if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp" -> "test.vcf.gz"), m)
+      ConfigUtils.mergeMaps(Map(
+        "multisample_variantcalling" -> multisampleCalling,
+        "single_sample_variantcalling" -> sampleCalling,
+        "library_variantcalling" -> libraryCalling,
+        "use_indel_realigner" -> realign,
+        "use_base_recalibration" -> baseRecalibration,
+        "sv_calling" -> svCalling,
+        "cnv_calling" -> cnvCalling,
+        "annotation" -> annotation,
+        "boot_runs" -> bootRuns), m)
+
+    }
+
+    if (!sample1 && !sample2) { // When no samples
+      intercept[IllegalArgumentException] {
+        initPipeline(map).script()
+      }
+      Logging.errors.clear()
+    } else {
+      val pipeline = initPipeline(map)
+      pipeline.script()
+
+      val numberLibs = (if (sample1) 1 else 0) + (if (sample2) 2 else 0)
+      val numberSamples = (if (sample1) 1 else 0) + (if (sample2) 1 else 0)
+
+      pipeline.functions.count(_.isInstanceOf[MarkDuplicates]) shouldBe (numberLibs + (if (sample2) 1 else 0))
+
+      // Gatk preprocess
+      pipeline.functions.count(_.isInstanceOf[IndelRealigner]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
+      pipeline.functions.count(_.isInstanceOf[RealignerTargetCreator]) shouldBe (numberLibs * (if (realign) 1 else 0) + (if (sample2 && realign) 1 else 0))
+      pipeline.functions.count(_.isInstanceOf[BaseRecalibrator]) shouldBe (if (dbsnp && baseRecalibration) (numberLibs * 2) else 0)
+      pipeline.functions.count(_.isInstanceOf[PrintReads]) shouldBe (if (dbsnp && baseRecalibration) numberLibs else 0)
+
+      pipeline.summarySettings.get("boot_runs") shouldBe Some(bootRuns.getOrElse(100))
+
+      pipeline.summaryFile shouldBe new File(BastyTest.outputDir, "Basty.summary.json")
+      pipeline.summaryFiles shouldBe Map()
+
+      pipeline.samples foreach {
+        case (sampleId, sample) =>
+          sample.summarySettings shouldBe Map()
+          sample.summaryFiles.get("variants_fasta") should not be None
+          sample.summaryFiles.get("consensus_fasta") should not be None
+          sample.summaryFiles.get("consensus_variants_fasta") should not be None
+          sample.summaryFiles.get("snps_only_variants_fasta") should not be None
+          sample.summaryFiles.get("snps_only_consensus_fasta") should not be None
+          sample.summaryFiles.get("snps_only_consensus_variants_fasta") should not be None
+          sample.summaryStats shouldBe Map()
+          sample.libraries.foreach {
+            case (libId, lib) =>
+              lib.summarySettings shouldBe Map()
+              lib.summaryFiles shouldBe Map()
+              lib.summaryStats shouldBe Map()
+          }
+      }
+
+      pipeline.functions.count(_.isInstanceOf[VcfStats]) shouldBe (
+        (if (multisampleCalling) 2 else 0) +
+        (if (sampleCalling) numberSamples * 2 else 0) +
+        (if (libraryCalling) numberLibs * 2 else 0))
+
+      pipeline.functions.count(_.isInstanceOf[BastyGenerateFasta]) shouldBe 2 + (2 * numberSamples)
+      pipeline.functions.count(_.isInstanceOf[Raxml]) shouldBe (2 * (2 + bootRuns.getOrElse(100)))
+      pipeline.functions.count(_.isInstanceOf[RunGubbins]) shouldBe 2
+    }
+  }
+}
+
+object BastyTest {
+  val outputDir = Files.createTempDir()
+  outputDir.deleteOnExit()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
+
+  private def copyFile(name: String): Unit = {
+    val is = getClass.getResourceAsStream("/" + name)
+    val os = new FileOutputStream(new File(outputDir, name))
+    org.apache.commons.io.IOUtils.copy(is, os)
+    os.close()
+  }
+
+  copyFile("ref.fa")
+  copyFile("ref.dict")
+  copyFile("ref.fa.fai")
+
+  val config = Map(
+    "name_prefix" -> "test",
+    "cache" -> true,
+    "dir" -> "test",
+    "vep_script" -> "test",
+    "output_dir" -> outputDir,
+    "reference_fasta" -> (outputDir + File.separator + "ref.fa"),
+    "gatk_jar" -> "test",
+    "samtools" -> Map("exe" -> "test"),
+    "bcftools" -> Map("exe" -> "test"),
+    "fastqc" -> Map("exe" -> "test"),
+    "input_alleles" -> "test",
+    "variantcallers" -> "raw",
+    "fastqc" -> Map("exe" -> "test"),
+    "seqtk" -> Map("exe" -> "test"),
+    "sickle" -> Map("exe" -> "test"),
+    "cutadapt" -> Map("exe" -> "test"),
+    "bwa" -> Map("exe" -> "test"),
+    "samtools" -> Map("exe" -> "test"),
+    "macs2" -> Map("exe" -> "test"),
+    "igvtools" -> Map("exe" -> "test", "igvtools_jar" -> "test"),
+    "wigtobigwig" -> Map("exe" -> "test"),
+    "md5sum" -> Map("exe" -> "test"),
+    "bgzip" -> Map("exe" -> "test"),
+    "tabix" -> Map("exe" -> "test"),
+    "breakdancerconfig" -> Map("exe" -> "test"),
+    "breakdancercaller" -> Map("exe" -> "test"),
+    "pindelconfig" -> Map("exe" -> "test"),
+    "pindelcaller" -> Map("exe" -> "test"),
+    "pindelvcf" -> Map("exe" -> "test"),
+    "clever" -> Map("exe" -> "test"),
+    "delly" -> Map("exe" -> "test"),
+    "rungubbins" -> Map("exe" -> "test"),
+    "raxml" -> Map("exe" -> "test"),
+    "pysvtools" -> Map(
+      "exe" -> "test",
+      "exclusion_regions" -> "test",
+      "translocations_only" -> false),
+    "freec" -> Map(
+      "exe" -> "test",
+      "chrFiles" -> "test",
+      "chrLenFile" -> "test"
+    )
+  )
+
+  val sample1 = Map(
+    "samples" -> Map("sample1" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
+      )
+    )
+    )))
+
+  val sample2 = Map(
+    "samples" -> Map("sample3" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
+      ),
+      "lib2" -> Map(
+        "R1" -> inputTouch("2_2_R1.fq"),
+        "R2" -> inputTouch("2_2_R2.fq")
+      )
+    )
+    )))
+}
\ No newline at end of file