diff --git a/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/RscriptCommandLineFunction.scala b/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/RscriptCommandLineFunction.scala index c773de6155b5a771f242dbbe83a4a21f98089eaa..0e4a29378a8a4334eeed795198c117431bdee8e2 100644 --- a/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/RscriptCommandLineFunction.scala +++ b/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/extensions/RscriptCommandLineFunction.scala @@ -15,13 +15,9 @@ */ package nl.lumc.sasc.biopet.core.extensions -import java.io.{ File, FileOutputStream } - import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction import nl.lumc.sasc.biopet.utils.rscript.Rscript -import scala.sys.process._ - /** * General rscript extension * diff --git a/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala b/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala index 02c860fdb1719c8c4635d467bea752b74745f9b2..bb6e1bf5606f21e99ceb7d557a93b68b690c0c3d 100644 --- a/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala +++ b/public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala @@ -184,6 +184,7 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config map.toMap } } + object WriteSummary { /** Retrive checksum from file */ def parseChecksum(checksumFile: File): String = { diff --git a/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala index 31b70db9d1595f2dd5614c9abe4a5fa1047ed834..79741a2c2eb73a39cce67d8ad48b5680ba939163 100644 --- a/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala +++ b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala @@ -19,6 +19,7 @@ class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar { @Test def testDefault: Unit = { + Logging.errors.clear() make(config :: testReferenceNoIndex :: Nil).referenceFasta() Logging.checkErrors(true) diff --git a/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummarizableTest.scala b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummarizableTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..1be487ff6d1309685d004d4b2f8f8b5523a93d75 --- /dev/null +++ b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummarizableTest.scala @@ -0,0 +1,32 @@ +package nl.lumc.sasc.biopet.core.summary + +import java.io.File + +import org.scalatest.Matchers +import org.scalatest.testng.TestNGSuite +import org.testng.annotations.Test + +/** + * Created by pjvanthof on 14/01/16. + */ +class SummarizableTest extends TestNGSuite with Matchers { + @Test + def testDefaultMerge: Unit = { + val summarizable = new Summarizable { + def summaryFiles: Map[String, File] = ??? + def summaryStats: Any = ??? + } + intercept[IllegalStateException] { + summarizable.resolveSummaryConflict("1", "1", "key") + } + } + + def testOverrideMerge: Unit = { + val summarizable = new Summarizable { + def summaryFiles: Map[String, File] = ??? + def summaryStats: Any = ??? + override def resolveSummaryConflict(v1: Any, v2: Any, key: String) = v1 + } + summarizable.resolveSummaryConflict("1", "1", "key") shouldBe "1" + } +} diff --git a/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScriptTest.scala b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScriptTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..a53fe068f9537cdce91a75f45ef9393631377d0d --- /dev/null +++ b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScriptTest.scala @@ -0,0 +1,126 @@ +package nl.lumc.sasc.biopet.core.summary + +import java.io.File + +import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile +import nl.lumc.sasc.biopet.core.extensions.Md5sum +import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable } +import org.broadinstitute.gatk.queue.QScript +import org.scalatest.Matchers +import org.scalatest.testng.TestNGSuite +import org.testng.annotations.Test +import SummaryQScriptTest._ + +/** + * Created by pjvanthof on 14/01/16. + */ +class SummaryQScriptTest extends TestNGSuite with Matchers { + @Test + def testNoJobs: Unit = { + SummaryQScript.md5sumCache.clear() + val script = makeQscript() + script.addSummaryJobs() + SummaryQScript.md5sumCache shouldBe empty + } + + @Test + def testFiles: Unit = { + SummaryQScript.md5sumCache.clear() + val file = new File(s".${File.separator}bla") + val script = makeQscript(files = Map("file" -> file)) + script.addSummaryJobs() + SummaryQScript.md5sumCache should not be empty + SummaryQScript.md5sumCache.toMap shouldBe Map( + new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5")) + script.functions.size shouldBe 2 + assert(script.functions + .filter(_.isInstanceOf[Md5sum]) + .map(_.asInstanceOf[Md5sum]) + .exists(_.cmdLine.contains(" || "))) + } + + @Test + def testDuplicateFiles: Unit = { + SummaryQScript.md5sumCache.clear() + val file = new File(s".${File.separator}bla") + val script = makeQscript(files = Map("file" -> file, "file2" -> file)) + script.addSummaryJobs() + SummaryQScript.md5sumCache should not be empty + SummaryQScript.md5sumCache.toMap shouldBe Map( + new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5")) + script.functions.size shouldBe 2 + assert(script.functions + .filter(_.isInstanceOf[Md5sum]) + .map(_.asInstanceOf[Md5sum]) + .exists(_.cmdLine.contains(" || "))) + } + + @Test + def testAddSummarizable: Unit = { + SummaryQScript.md5sumCache.clear() + val file = new File(s".${File.separator}bla") + val script = makeQscript() + script.addSummarizable(makeSummarizable(files = Map("file" -> file, "file2" -> file)), "test") + script.summarizables.size shouldBe 1 + script.addSummaryJobs() + SummaryQScript.md5sumCache should not be empty + SummaryQScript.md5sumCache.toMap shouldBe Map( + new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5")) + script.functions.size shouldBe 2 + assert(script.functions + .filter(_.isInstanceOf[Md5sum]) + .map(_.asInstanceOf[Md5sum]) + .exists(_.cmdLine.contains(" || "))) + } + + @Test + def testInputFile: Unit = { + SummaryQScript.md5sumCache.clear() + val file = new File(s".${File.separator}bla") + val script = makeQscript() + script.addSummarizable(makeSummarizable(files = Map("file" -> file, "file2" -> file)), "test") + script.summarizables.size shouldBe 1 + script.inputFiles :+= InputFile(file, Some("md5sum")) + script.inputFiles :+= InputFile(file, None) + script.addSummaryJobs() + SummaryQScript.md5sumCache should not be empty + SummaryQScript.md5sumCache.toMap shouldBe Map( + new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5")) + script.functions.size shouldBe 3 + assert(script.functions + .filter(_.isInstanceOf[Md5sum]) + .map(_.asInstanceOf[Md5sum]) + .exists(_.cmdLine.contains(" || "))) + } + + @Test + def testAddQscript: Unit = { + SummaryQScript.md5sumCache.clear() + val script = makeQscript() + script.addSummaryQScript(script) + script.summaryQScripts.head shouldBe script + } +} + +object SummaryQScriptTest { + def makeQscript(settings: Map[String, Any] = Map(), + files: Map[String, File] = Map(), + c: Map[String, Any] = Map()) = + new SummaryQScript with QScript { + outputDir = new File(".") + override def globalConfig = new Config(c) + def summarySettings: Map[String, Any] = settings + def summaryFiles: Map[String, File] = files + val tempFile = File.createTempFile("summary", ".json") + tempFile.deleteOnExit() + def summaryFile: File = tempFile + def init(): Unit = ??? + def biopetScript(): Unit = ??? + def root: Configurable = null + } + + def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable { + def summaryFiles: Map[String, File] = files + def summaryStats: Any = stats + } +} \ No newline at end of file diff --git a/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..4bb196c162ddb93c85b19d1364d34789fb77e8b9 --- /dev/null +++ b/public/biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala @@ -0,0 +1,346 @@ +package nl.lumc.sasc.biopet.core.summary + +import java.io.{ PrintWriter, File } + +import com.google.common.io.Files +import nl.lumc.sasc.biopet.core._ +import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable } +import nl.lumc.sasc.biopet.utils.summary.Summary +import org.broadinstitute.gatk.queue.function.CommandLineFunction +import org.broadinstitute.gatk.queue.{ QScript, QSettings } +import org.scalatest.Matchers +import org.scalatest.testng.TestNGSuite +import WriteSummaryTest._ +import org.testng.annotations.Test + +import scala.util.matching.Regex + +/** + * Created by pjvanthof on 15/01/16. + */ +class WriteSummaryTest extends TestNGSuite with Matchers { + + @Test + def testWrongRoot(): Unit = { + intercept[IllegalArgumentException] { + makeWriter(null) + } + } + + /** This is a basic summary test, no matter the content this should always be true */ + def basicSummaryTest(summary: Summary, + name: String, + sampleId: Option[String] = None, + libId: Option[String] = None): Unit = { + summary.getValue(sampleId, libId, name) should not be None + summary.getValue(sampleId, libId, name, "files", "pipeline").get shouldBe a[Map[_, _]] + summary.getValue(sampleId, libId, name, "settings").get shouldBe a[Map[_, _]] + summary.getValue(sampleId, libId, name, "executables").get shouldBe a[Map[_, _]] + + summary.getValue("meta") should not be None + summary.getValue("meta", "pipeline_name") shouldBe Some(name) + summary.getValue("meta", "last_commit_hash") shouldBe Some(nl.lumc.sasc.biopet.LastCommitHash) + summary.getValue("meta", "pipeline_version") shouldBe Some(nl.lumc.sasc.biopet.Version) + summary.getValue("meta", "output_dir") shouldBe Some(new File(".").getAbsolutePath) + summary.getValue("meta", "summary_creation") should not be None + } + + def createFakeCheckSum(file: File): Unit = { + file.getParentFile.mkdirs() + val writer = new PrintWriter(file) + writer.println("checksum file") + writer.close() + file.deleteOnExit() + } + + @Test + def testEmpty(): Unit = { + val qscript = makeQscript(name = "test") + val writer = makeWriter(qscript) + writer.freezeFieldValues() + writer.deps shouldBe empty + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + } + + @Test + def testMergeQscript(): Unit = { + val qscript = makeQscript(name = "test") + val qscript2 = makeQscript(name = "test2") + qscript.addSummaryQScript(qscript2) + val summaryWriter = new PrintWriter(qscript2.summaryFile) + summaryWriter.println("""{ "test2": "value" }""") + summaryWriter.close() + val writer = makeWriter(qscript) + writer.freezeFieldValues() + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + summary.getValue("test2") shouldBe Some("value") + } + + @Test + def testSingleJob(): Unit = { + val qscript = makeQscript("test") + val writer = makeWriter(qscript) + val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + } + + @Test + def testSingleJavaJob(): Unit = { + val qscript = makeQscript("test") + val writer = makeWriter(qscript) + val summarizable = makeJavaCommand(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.add(summarizable) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version") + } + + @Test + def testVersion(): Unit = { + val qscript = makeQscript("test") + val writer = makeWriter(qscript) + val summarizable = makeVersionSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.add(summarizable) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version") + } + + @Test + def testSampleLibrary(): Unit = { + val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName")) + val writer = makeWriter(qscript) + val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.deps shouldBe empty + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName")) + summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + } + + @Test + def testSample(): Unit = { + val qscript = makeSampleLibraryQscript("test", s = Some("sampleName")) + val writer = makeWriter(qscript) + val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.deps shouldBe empty + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None) + summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + } + + @Test + def testMultisampleQscript(): Unit = { + val qscript = makeMultisampleQscript("test", multisampleConfig) + val writer = makeWriter(qscript) + val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value")) + qscript.addSummarizable(summarizable, "tool_1") + qscript.addSummaryJobs() + createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla"))) + writer.freezeFieldValues() + writer.deps shouldBe empty + writer.run() + + val summary = new Summary(writer.out) + basicSummaryTest(summary, "test") + summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value") + summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum") + + summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None + } + +} + +object WriteSummaryTest { + def makeWriter(root: Configurable, c: Map[String, Any] = Map()) = new WriteSummary(root) { + override def globalConfig = new Config(c) + override def outputs = Seq() + override def inputs = Seq() + qSettings = new QSettings { + jobName = "test" + jobTempDir = Files.createTempDir() + jobTempDir.deleteOnExit() + jobPriority = Some(1) + } + override def absoluteCommandDirectory() {} + } + + def makeQscript(name: String, + settings: Map[String, Any] = Map(), + files: Map[String, File] = Map(), + c: Map[String, Any] = Map()) = + new SummaryQScript with QScript { + summaryName = name + outputDir = new File(".").getAbsoluteFile + override def globalConfig = new Config(c) + def summarySettings: Map[String, Any] = settings + def summaryFiles: Map[String, File] = files + val tempFile = File.createTempFile("summary", ".json") + tempFile.deleteOnExit() + def summaryFile: File = tempFile + def init(): Unit = {} + def biopetScript(): Unit = {} + def root: Configurable = null + } + + def makeSampleLibraryQscript(name: String, + settings: Map[String, Any] = Map(), + files: Map[String, File] = Map(), + c: Map[String, Any] = Map(), + s: Option[String] = None, + l: Option[String] = None) = + new SummaryQScript with QScript with SampleLibraryTag { + sampleId = s + libId = l + summaryName = "test" + outputDir = new File(".").getAbsoluteFile + override def globalConfig = new Config(c) + def summarySettings: Map[String, Any] = settings + def summaryFiles: Map[String, File] = files + val tempFile = File.createTempFile("summary", ".json") + tempFile.deleteOnExit() + def summaryFile: File = tempFile + def init(): Unit = {} + def biopetScript(): Unit = {} + def root: Configurable = null + } + + def makeMultisampleQscript(name: String, + c: Map[String, Any], + settings: Map[String, Any] = Map(), + files: Map[String, File] = Map()) = + new MultiSampleQScript with QScript { + summaryName = "test" + outputDir = new File(".").getAbsoluteFile + override def globalConfig = new Config(c) + def summarySettings: Map[String, Any] = settings + def summaryFiles: Map[String, File] = files + val tempFile = File.createTempFile("summary", ".json") + tempFile.deleteOnExit() + def summaryFile: File = tempFile + def init(): Unit = {} + def biopetScript(): Unit = {} + def root: Configurable = null + + class Sample(id: String) extends AbstractSample(id) { + class Library(id: String) extends AbstractLibrary(id) { + protected def addJobs(): Unit = {} + def summaryFiles: Map[String, File] = files + def summaryStats: Any = Map() + } + + def makeLibrary(id: String): Library = new Library(id) + protected def addJobs(): Unit = {} + def summaryFiles: Map[String, File] = files + def summaryStats: Any = Map() + } + + def makeSample(id: String): Sample = new Sample(id) + + def addMultiSampleJobs(): Unit = {} + } + + val multisampleConfig = Map("samples" -> Map("sampleName" -> Map("libraries" -> Map("libName" -> Map())))) + + def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable { + def summaryFiles: Map[String, File] = files + def summaryStats: Any = stats + } + + def makeJavaCommand(files: Map[String, File] = Map(), + stats: Map[String, Any] = Map(), + c: Map[String, Any] = Map()) = new BiopetJavaCommandLineFunction with Summarizable with Version { + override def globalConfig = new Config(c) + override def configName = "java_command" + def root: Configurable = null + def summaryStats: Map[String, Any] = stats + def summaryFiles: Map[String, File] = files + + def versionCommand: String = "echo test version" + def versionRegex: Regex = """(.*)""".r + override def getVersion = Some("test version") + + override def outputs = Seq() + override def inputs = Seq() + qSettings = new QSettings { + jobName = "test" + jobTempDir = Files.createTempDir() + jobTempDir.deleteOnExit() + jobPriority = Some(1) + } + override def absoluteCommandDirectory() {} + } + + def makeVersionSummarizable(files: Map[String, File] = Map(), + stats: Map[String, Any] = Map(), + c: Map[String, Any] = Map()) = + new CommandLineFunction with Configurable with Summarizable with Version { + override def globalConfig = new Config(c) + override def configName = "version_command" + def root: Configurable = null + + def summaryFiles: Map[String, File] = files + def summaryStats: Any = stats + + def versionCommand: String = "echo test version" + def versionRegex: Regex = """(.*)""".r + override def getVersion = Some("test version") + + def commandLine: String = "" + + override def outputs = Seq() + override def inputs = Seq() + qSettings = new QSettings { + jobName = "test" + jobTempDir = Files.createTempDir() + jobTempDir.deleteOnExit() + jobPriority = Some(1) + } + override def absoluteCommandDirectory() {} + } + +} \ No newline at end of file diff --git a/public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/Logging.scala b/public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/Logging.scala index d01755e07300cf09f874bc36f07d640d6567c84e..9d4b9dc2bf84bcda084ce5de9647620e1a130658 100644 --- a/public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/Logging.scala +++ b/public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/Logging.scala @@ -36,7 +36,7 @@ trait Logging { object Logging { val logger = Logger.getRootLogger - private val errors: ListBuffer[Exception] = ListBuffer() + private[biopet] val errors: ListBuffer[Exception] = ListBuffer() def addError(error: String, debug: String = null): Unit = { val msg = error + (if (debug != null && logger.isDebugEnabled) "; " + debug else "")