Merge branch 'feature-sage_unit_tests' into 'develop'

Added basic testing to sage



See merge request !457

biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/PrefixFastq.scala

@@ -48,20 +48,3 @@ class PrefixFastq(val root: Configurable) extends ToolCommandFunction {
     required("-o", outputFastq) +
     optional("-s", prefixSeq)
 }
-
-object PrefixFastq {
-  /**
-   * Create a PrefixFastq class object with a sufix ".prefix.fastq" in the output folder
-   *
-   * @param root parent object
-   * @param input input file
-   * @param outputDir outputFolder
-   * @return PrefixFastq class object
-   */
-  def apply(root: Configurable, input: File, outputDir: String): PrefixFastq = {
-    val prefixFastq = new PrefixFastq(root)
-    prefixFastq.inputFastq = input
-    prefixFastq.outputFastq = new File(outputDir, input.getName + ".prefix.fastq")
-    prefixFastq
-  }
-}

sage/pom.xml

@@ -42,5 +42,17 @@
             <artifactId>Mapping</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.8</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.10</artifactId>
+            <version>2.2.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

sage/src/main/scala/nl/lumc/sasc/biopet/pipelines/sage/Sage.scala

@@ -23,7 +23,7 @@ import nl.lumc.sasc.biopet.pipelines.flexiprep.Flexiprep
 import nl.lumc.sasc.biopet.pipelines.mapping.Mapping
 import nl.lumc.sasc.biopet.extensions.tools.SquishBed
 import nl.lumc.sasc.biopet.extensions.tools.{ BedtoolsCoverageToCounts, PrefixFastq, SageCountFastq, SageCreateLibrary, SageCreateTagCounts }
-import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.Logging
 import org.broadinstitute.gatk.queue.QScript
 
 class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
@@ -54,26 +54,20 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
 
   def summaryFile: File = new File(outputDir, "Sage.summary.json")
 
-  //TODO: Add summary
   def summaryFiles: Map[String, File] = Map()
 
-  //TODO: Add summary
   def summarySettings: Map[String, Any] = Map()
 
   def makeSample(id: String) = new Sample(id)
   class Sample(sampleId: String) extends AbstractSample(sampleId) {
-    //TODO: Add summary
     def summaryFiles: Map[String, File] = Map()
 
-    //TODO: Add summary
     def summaryStats: Map[String, Any] = Map()
 
     def makeLibrary(id: String) = new Library(id)
     class Library(libId: String) extends AbstractLibrary(libId) {
-      //TODO: Add summary
       def summaryFiles: Map[String, File] = Map()
 
-      //TODO: Add summary
       def summaryStats: Map[String, Any] = Map()
 
       val inputFastq: File = config("R1")
@@ -92,9 +86,7 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
 
         flexiprep.outputDir = new File(libDir, "flexiprep/")
         flexiprep.inputR1 = inputFastq
-        flexiprep.init()
-        flexiprep.biopetScript()
-        qscript.addAll(flexiprep.functions)
+        add(flexiprep)
 
         val flexiprepOutput = for ((key, file) <- flexiprep.outputFiles if key.endsWith("output_R1")) yield file
         val pf = new PrefixFastq(qscript)
@@ -106,9 +98,7 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
 
         mapping.inputR1 = pf.outputFastq
         mapping.outputDir = libDir
-        mapping.init()
-        mapping.biopetScript()
-        qscript.addAll(mapping.functions)
+        add(mapping)
 
         if (config("library_counts", default = false).asBoolean) {
           addBedtoolsCounts(mapping.finalBamFile, sampleId + "-" + libId, libDir)
@@ -123,17 +113,17 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
       val libraryFastqFiles = libraries.map(_._2.prefixFastq).toList
 
       val bamFile: File = if (libraryBamfiles.size == 1) libraryBamfiles.head
-      else if (libraryBamfiles.size > 1) {
+      else {
         val mergeSamFiles = MergeSamFiles(qscript, libraryBamfiles, new File(sampleDir, s"$sampleId.bam"))
         qscript.add(mergeSamFiles)
         mergeSamFiles.output
-      } else null
+      }
       val fastqFile: File = if (libraryFastqFiles.size == 1) libraryFastqFiles.head
-      else if (libraryFastqFiles.size > 1) {
+      else {
         val cat = Cat(qscript, libraryFastqFiles, createFile(".fastq"))
         qscript.add(cat)
         cat.output
-      } else null
+      }
 
       addBedtoolsCounts(bamFile, sampleId, sampleDir)
       addTablibCounts(fastqFile, sampleId, sampleDir)
@@ -142,21 +132,21 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
 
   def init() {
     if (transcriptome.isEmpty && tagsLibrary.isEmpty)
-      throw new IllegalStateException("No transcriptome or taglib found")
+      Logging.addError("No transcriptome or taglib found")
     if (countBed.isEmpty)
-      throw new IllegalStateException("No bedfile supplied, please add a countBed")
+      Logging.addError("No bedfile supplied, please add a countBed")
   }
 
   def biopetScript() {
     val squishBed = new SquishBed(this)
-    squishBed.input = countBed.get
-    squishBed.output = new File(outputDir, countBed.get.getName.stripSuffix(".bed") + ".squish.bed")
+    squishBed.input = countBed.getOrElse(null)
+    squishBed.output = new File(outputDir, countBed.getOrElse(new File("fake")).getName.stripSuffix(".bed") + ".squish.bed")
     add(squishBed)
     squishedCountBed = squishBed.output
 
     if (tagsLibrary.isEmpty) {
       val cdl = new SageCreateLibrary(this)
-      cdl.input = transcriptome.get
+      cdl.input = transcriptome.getOrElse(null)
       cdl.output = new File(outputDir, "taglib/tag.lib")
       cdl.noAntiTagsOutput = new File(outputDir, "taglib/no_antisense_genes.txt")
       cdl.noTagsOutput = new File(outputDir, "taglib/no_sense_genes.txt")

sage/src/test/resources/ref.dict

+@HD	VN:1.4	SO:unsorted
+@SQ	SN:chr1	LN:9	UR:file:/home/pjvan_thof/pipelines/biopet/public/mapping/src/test/resources/ref.fa	M5:fe15dbbd0900310caf32827f6da57550

sage/src/test/resources/ref.fa

+>chr1
+AGTAGTAGT

sage/src/test/resources/ref.fa.fai

+chr1	9	6	9	10

sage/src/test/scala/nl/lumc/sasc/biopet/pipelines/sage/SageTest.scala

+package nl.lumc.sasc.biopet.pipelines.sage
+
+import java.io.{ File, FileOutputStream }
+
+import com.google.common.io.Files
+import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
+import nl.lumc.sasc.biopet.utils.config.Config
+import org.broadinstitute.gatk.queue.QSettings
+import org.scalatest.Matchers
+import org.scalatest.testng.TestNGSuite
+import org.testng.annotations.{ DataProvider, Test }
+
+/**
+ * Created by pjvanthof on 28/09/16.
+ */
+class SageTest extends TestNGSuite with Matchers {
+  def initPipeline(map: Map[String, Any]): Sage = {
+    new Sage() {
+      override def configNamespace = "sage"
+
+      override def globalConfig = new Config(ConfigUtils.mergeMaps(map, SageTest.config))
+
+      qSettings = new QSettings
+      qSettings.runName = "test"
+    }
+  }
+
+  @DataProvider(name = "sageOptions")
+  def sageOptions = {
+    for (
+      s1 <- sample1; s2 <- sample2
+    ) yield Array("", s1, s2)
+  }
+
+  def sample1 = Array(false, true)
+  def sample2 = Array(false, true)
+  def transcriptome = true
+  def countBed = true
+  def tagsLibrary = false
+  def libraryCounts: Option[Boolean] = None
+
+  @Test(dataProvider = "sageOptions")
+  def testSage(f: String, sample1: Boolean, sample2: Boolean): Unit = {
+    val map = {
+      var m: Map[String, Any] = SageTest.config
+      if (sample1) m = ConfigUtils.mergeMaps(SageTest.sample1, m)
+      if (sample2) m = ConfigUtils.mergeMaps(SageTest.sample2, m)
+      ConfigUtils.mergeMaps(
+        (if (transcriptome) Map[String, Any]("transcriptome" -> SageTest.inputTouch("trans.fa")) else Map[String, Any]()) ++
+          (if (countBed) Map[String, Any]("count_bed" -> SageTest.inputTouch("count.bed")) else Map[String, Any]()) ++
+          (if (tagsLibrary) Map[String, Any]("tags_library" -> SageTest.inputTouch("tablib")) else Map[String, Any]()) ++
+          libraryCounts.map("library_counts" -> _),
+        m)
+
+    }
+
+    if ((!sample1 && !sample2) || !countBed || (!transcriptome && !tagsLibrary)) {
+      // When no samples
+      intercept[IllegalStateException] {
+        initPipeline(map).script()
+      }
+      Logging.errors.clear()
+    } else {
+      val pipeline = initPipeline(map)
+      pipeline.script()
+
+      val numberLibs = (if (sample1) 1 else 0) + (if (sample2) 2 else 0)
+      val numberSamples = (if (sample1) 1 else 0) + (if (sample2) 1 else 0)
+
+      pipeline.summaryFile shouldBe new File(SageTest.outputDir, "Sage.summary.json")
+      pipeline.summaryFiles shouldBe Map()
+      pipeline.summarySettings shouldBe Map()
+
+      pipeline.samples.foreach {
+        case (sampleId, sample) =>
+          sample.summaryFiles shouldBe Map()
+          sample.summaryStats shouldBe Map()
+          sample.summarySettings shouldBe Map()
+          sample.libraries.foreach {
+            case (libId, lib) =>
+              lib.summaryFiles shouldBe Map()
+              lib.summaryStats shouldBe Map()
+              lib.summarySettings shouldBe Map()
+          }
+      }
+
+    }
+  }
+}
+
+class SageNoBedTest extends SageTest {
+  override def sample1 = Array(true)
+  override def sample2 = Array(false)
+  override def countBed = false
+}
+
+class SageNoLibTest extends SageTest {
+  override def sample1 = Array(true)
+  override def sample2 = Array(false)
+  override def transcriptome = false
+  override def tagsLibrary = false
+}
+
+class SageLibraryCountsTest extends SageTest {
+  override def sample1 = Array(true)
+  override def sample2 = Array(false)
+  override def libraryCounts = Some(true)
+}
+
+object SageTest {
+  val outputDir = Files.createTempDir()
+  outputDir.deleteOnExit()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
+
+  inputTouch("ref.ebwt")
+
+  private def copyFile(name: String): Unit = {
+    val is = getClass.getResourceAsStream("/" + name)
+    val os = new FileOutputStream(new File(outputDir, name))
+    org.apache.commons.io.IOUtils.copy(is, os)
+    os.close()
+  }
+
+  copyFile("ref.fa")
+  copyFile("ref.dict")
+  copyFile("ref.fa.fai")
+
+  val config: Map[String, Any] = Map(
+    "reference_fasta" -> (outputDir + File.separator + "ref.fa"),
+    "output_dir" -> outputDir.getAbsolutePath,
+    "fastqc" -> Map("exe" -> "test"),
+    "seqtk" -> Map("exe" -> "test"),
+    "md5sum" -> Map("exe" -> "test"),
+    "bedtools" -> Map("exe" -> "test"),
+    "bowtie" -> Map("exe" -> "test"),
+    "bowtie_index" -> inputTouch("ref")
+  )
+
+  val sample1 = Map(
+    "samples" -> Map("sample1" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
+      )
+    )
+    )))
+
+  val sample2 = Map(
+    "samples" -> Map("sample3" -> Map("libraries" -> Map(
+      "lib1" -> Map(
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
+      ),
+      "lib2" -> Map(
+        "R1" -> inputTouch("2_2_R1.fq"),
+        "R2" -> inputTouch("2_2_R2.fq")
+      )
+    )
+    )))
+
+}
\ No newline at end of file