Fixed all core unit tests

basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala

@@ -36,7 +36,7 @@ class BastyTest extends TestNGSuite with Matchers {
   def initPipeline(map: Map[String, Any]): Basty = {
     new Basty() {
       override def configNamespace = "shiva"
-      override def globalConfig = new Config(ConfigUtils.mergeMaps(map, BastyTest.config))
+      override def globalConfig = new Config(map)
       qSettings = new QSettings
       qSettings.runName = "test"
     }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala

@@ -38,6 +38,8 @@ import scala.concurrent.ExecutionContext.Implicits.global
 class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Configurable {
   this.analysisName = getClass.getSimpleName
 
+  require(parent != null)
+
   /** To access qscript for this summary */
   val qscript = parent
 

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilderTest.scala

@@ -18,10 +18,14 @@ import java.io.File
 import java.nio.file.Paths
 
 import com.google.common.io.Files
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.Test
 
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
 /**
  * Created by pjvanthof on 24/02/16.
  */
@@ -41,10 +45,26 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
 
       def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = ReportPage(Nil, Nil, Map())
     }
+    val dbFile = File.createTempFile("summary.", ".db")
+    dbFile.deleteOnExit()
+    val db = SummaryDb.openSqliteSummary(dbFile)
+    db.createTables()
+
+    val sample = Some("sampleName")
+    val lib = Some("libName")
+
+    sample.foreach { sampleName =>
+      val sampleId = Await.result(db.createSample(sampleName, 0), Duration.Inf)
+      lib.foreach { libName =>
+        Await.result(db.createLibrary(libName, 0, sampleId), Duration.Inf)
+      }
+
+    }
+
 
     val tempDir = Files.createTempDir()
     tempDir.deleteOnExit()
-    val args = Array("-s", resourcePath("/empty_summary.json"), "-o", tempDir.getAbsolutePath)
+    val args = Array("-s", dbFile.getAbsolutePath, "-o", tempDir.getAbsolutePath)
     builder.main(args)
     builder.extFiles.foreach(x => new File(tempDir, "ext" + File.separator + x.targetPath) should exist)
 
@@ -55,6 +75,8 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
     createFile("Samples", "sampleName", "index.html") should exist
     createFile("Samples", "sampleName", "Libraries", "index.html") should exist
     createFile("Samples", "sampleName", "Libraries", "libName", "index.html") should exist
+
+    db.close()
   }
 
 }

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala

@@ -18,9 +18,13 @@ import java.io.File
 import java.nio.file.Paths
 
 import com.google.common.io.Files
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
-import org.testng.annotations.{ DataProvider, Test }
+import org.testng.annotations.{DataProvider, Test}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
 
 /**
  * Created by pjvanthof on 24/02/16.
@@ -36,7 +40,7 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
     val sample = Array(Some("sampleName"), None)
     val lib = Array(Some("libName"), None)
     val nested = Array(false, true)
-    for (s <- sample; l <- lib; n <- nested) yield Array(s, l, n)
+    for (s <- sample; l <- lib; n <- nested if (!(l.isDefined && s.isEmpty))) yield Array(s, l, n)
   }
 
   @Test(dataProvider = "testGeneratePages")
@@ -47,17 +51,30 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
         (if (nested) "p1" -> ReportPage(Nil, Nil, Map()) :: Nil else Nil), Nil, Map())
     }
 
+    val dbFile = File.createTempFile("summary.", ".db")
+    dbFile.deleteOnExit()
+    val db = SummaryDb.openSqliteSummary(dbFile)
+    db.createTables()
+    sample.foreach { sampleName =>
+      val sampleId = Await.result(db.createSample(sampleName, 0), Duration.Inf)
+      lib.foreach { libName =>
+        Await.result(db.createLibrary(libName, 0, sampleId), Duration.Inf)
+      }
+    }
+
     val tempDir = Files.createTempDir()
     tempDir.deleteOnExit()
-    val args = Array("-s", resourcePath("/empty_summary.json"), "-o", tempDir.getAbsolutePath) ++
+    val args = Array("-s", dbFile.getAbsolutePath, "-o", tempDir.getAbsolutePath) ++
       sample.map(x => Array("-a", s"sampleId=$x")).getOrElse(Array()) ++
       lib.map(x => Array("-a", s"libId=$x")).getOrElse(Array())
     builder.main(args)
-    builder.sampleId shouldBe sample
-    builder.libId shouldBe lib
+    builder.sampleId shouldBe sample.flatMap(s => Await.result(db.getSampleId(0, s), Duration.Inf))
+    builder.libId shouldBe lib.flatMap(l => Await.result(db.getLibraryId(0, builder.sampleId.get, l), Duration.Inf))
     builder.extFiles.foreach(x => new File(tempDir, "ext" + File.separator + x.targetPath) should exist)
     new File(tempDir, "index.html") should exist
     new File(tempDir, "p1" + File.separator + "index.html").exists() shouldBe nested
+
+    db.close()
   }
 
   @Test

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala

@@ -14,19 +14,23 @@
  */
 package nl.lumc.sasc.biopet.core.summary
 
-import java.io.{ PrintWriter, File }
+import java.io.{File, PrintWriter}
 
 import com.google.common.io.Files
 import nl.lumc.sasc.biopet.core._
-import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
+import nl.lumc.sasc.biopet.utils.config.{Config, Configurable}
 import nl.lumc.sasc.biopet.utils.summary.Summary
 import org.broadinstitute.gatk.queue.function.CommandLineFunction
-import org.broadinstitute.gatk.queue.{ QScript, QSettings }
+import org.broadinstitute.gatk.queue.{QScript, QSettings}
 import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import WriteSummaryTest._
-import org.testng.annotations.Test
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
+import org.apache.commons.io.FileUtils
+import org.testng.annotations.{AfterClass, Test}
 
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
 import scala.util.matching.Regex
 
 /**
@@ -41,22 +45,19 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     }
   }
 
+  private var dirs: List[File] = Nil
+
   /** This is a basic summary test, no matter the content this should always be true */
-  def basicSummaryTest(summary: Summary,
-                       name: String,
+  def basicSummaryTest(summary: SummaryDb,
+                       runId: Int,
+                       dir: File,
                        sampleId: Option[String] = None,
                        libId: Option[String] = None): Unit = {
-    summary.getValue(sampleId, libId, name) should not be None
-    summary.getValue(sampleId, libId, name, "files", "pipeline").get shouldBe a[Map[_, _]]
-    summary.getValue(sampleId, libId, name, "settings").get shouldBe a[Map[_, _]]
-    summary.getValue(sampleId, libId, name, "executables").get shouldBe a[Map[_, _]]
-
-    summary.getValue("meta") should not be None
-    summary.getValue("meta", "pipeline_name") shouldBe Some(name)
-    summary.getValue("meta", "last_commit_hash") shouldBe Some(nl.lumc.sasc.biopet.LastCommitHash)
-    summary.getValue("meta", "pipeline_version") shouldBe Some(nl.lumc.sasc.biopet.Version)
-    summary.getValue("meta", "output_dir") shouldBe Some(new File(".").getAbsolutePath)
-    summary.getValue("meta", "summary_creation") should not be None
+    val run = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
+    run.commitHash shouldBe nl.lumc.sasc.biopet.LastCommitHash
+    run.version shouldBe nl.lumc.sasc.biopet.Version
+    run.outputDir shouldBe dir.getAbsolutePath
+
   }
 
   def createFakeCheckSum(file: File): Unit = {
@@ -69,38 +70,24 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
 
   @Test
   def testEmpty(): Unit = {
-    val qscript = makeQscript(name = "test")
+    val dir = Files.createTempDir()
+    dirs :+= dir
+    val qscript = makeQscript(name = "test", dir = dir)
     val writer = makeWriter(qscript)
     writer.freezeFieldValues()
     writer.deps shouldBe empty
     writer.run()
 
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-  }
-
-  @Test
-  def testMergeQscript(): Unit = {
-    val qscript = makeQscript(name = "test")
-    val qscript2 = makeQscript(name = "test2")
-    qscript.addSummaryQScript(qscript2)
-    val summaryWriter = new PrintWriter(qscript2.summaryFile)
-    summaryWriter.println("""{ "test2": "value" }""")
-    summaryWriter.close()
-    val writer = makeWriter(qscript)
-    writer.freezeFieldValues()
-    writer.run()
-
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-    //    summary.getValue("test2") shouldBe Some("value")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
   }
 
   @Test
   def testSingleJob(): Unit = {
-    val qscript = makeQscript("test")
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeQscript("test", dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.addSummarizable(summarizable, "tool_1")
@@ -109,17 +96,18 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     writer.freezeFieldValues()
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-    //    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
+    summary.getStatKeys(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), keyValues = Map("key" ->  List("key"))) shouldBe Map("key" -> Some("value"))
+    Await.result(summary.getFile(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), None, None, key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
   }
 
   @Test
   def testSingleJavaJob(): Unit = {
-    val qscript = makeQscript("test")
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeQscript("test", dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeJavaCommand(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.add(summarizable)
@@ -129,18 +117,18 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     writer.freezeFieldValues()
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-    //    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
-    //    summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
+    summary.getStatKeys(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), keyValues = Map("key" ->  List("key"))) shouldBe Map("key" -> Some("value"))
+    Await.result(summary.getFile(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), None, None, key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
   }
 
   @Test
   def testVersion(): Unit = {
-    val qscript = makeQscript("test")
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeQscript("test", dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeVersionSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.add(summarizable)
@@ -150,18 +138,18 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     writer.freezeFieldValues()
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-    //    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
-    //    summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
+    summary.getStatKeys(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), keyValues = Map("key" ->  List("key"))) shouldBe Map("key" -> Some("value"))
+    Await.result(summary.getFile(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), None, None, key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
   }
 
   @Test
   def testSampleLibrary(): Unit = {
-    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName"))
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName"), dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.addSummarizable(summarizable, "tool_1")
@@ -171,37 +159,38 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     writer.deps shouldBe empty
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
-    //    summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
+    summary.getStatKeys(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), Some(Right("sampleName")), Some(Right("libName")), keyValues = Map("key" ->  List("key"))) shouldBe Map("key" -> Some("value"))
+    Await.result(summary.getFile(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), Some(Right("sampleName")), Some(Right("libName")), key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
   }
 
   @Test
   def testSample(): Unit = {
-    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"))
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.addSummarizable(summarizable, "tool_1")
     qscript.addSummaryJobs()
     createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
+    writer.init()
     writer.freezeFieldValues()
     writer.deps shouldBe empty
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
-    //    summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
   }
 
   @Test
   def testMultisampleQscript(): Unit = {
-    val qscript = makeMultisampleQscript("test", multisampleConfig)
+    val dir = Files.createTempDir()
+    dirs :+= dir
+
+    val qscript = makeMultisampleQscript("test", multisampleConfig, dir = dir)
     val writer = makeWriter(qscript)
     val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
     qscript.addSummarizable(summarizable, "tool_1")
@@ -211,16 +200,16 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
     writer.deps shouldBe empty
     writer.run()
 
-    throw new Exception("Fix me")
-
-    //    val summary = new Summary(writer.out)
-    //    basicSummaryTest(summary, "test")
-    //    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
-    //    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
-    //
-    //    summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
+    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
+    basicSummaryTest(summary, qscript.summaryRunId, dir)
+    summary.getStatKeys(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), keyValues = Map("key" ->  List("key"))) shouldBe Map("key" -> Some("value"))
+    Await.result(summary.getFile(qscript.summaryRunId, Right("test"), Some(Right("tool_1")), None, None, key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
   }
 
+  @AfterClass
+  def removeDirs: Unit = {
+    dirs.foreach(FileUtils.deleteDirectory)
+  }
 }
 
 object WriteSummaryTest {
@@ -240,10 +229,11 @@ object WriteSummaryTest {
   def makeQscript(name: String,
                   settings: Map[String, Any] = Map(),
                   files: Map[String, File] = Map(),
-                  c: Map[String, Any] = Map()) =
+                  c: Map[String, Any] = Map(),
+                  dir: File) =
     new SummaryQScript with QScript {
       summaryName = name
-      outputDir = new File(".").getAbsoluteFile
+      outputDir = dir
       override def globalConfig = new Config(c)
       def summarySettings: Map[String, Any] = settings
       def summaryFiles: Map[String, File] = files
@@ -260,12 +250,13 @@ object WriteSummaryTest {
                                files: Map[String, File] = Map(),
                                c: Map[String, Any] = Map(),
                                s: Option[String] = None,
-                               l: Option[String] = None) =
+                               l: Option[String] = None,
+                               dir: File) =
     new SummaryQScript with QScript with SampleLibraryTag {
       sampleId = s
       libId = l
       summaryName = "test"
-      outputDir = new File(".").getAbsoluteFile
+      outputDir = dir
       override def globalConfig = new Config(c + ("exe" -> "test"))
       def summarySettings: Map[String, Any] = settings
       def summaryFiles: Map[String, File] = files
@@ -280,10 +271,11 @@ object WriteSummaryTest {
   def makeMultisampleQscript(name: String,
                              c: Map[String, Any],
                              settings: Map[String, Any] = Map(),
-                             files: Map[String, File] = Map()) =
+                             files: Map[String, File] = Map(),
+                             dir: File) =
     new MultiSampleQScript with QScript {
       summaryName = "test"
-      outputDir = new File(".").getAbsoluteFile
+      outputDir = dir
       override def globalConfig = new Config(c + ("exe" -> "test"))
       def summarySettings: Map[String, Any] = settings
       def summaryFiles: Map[String, File] = files