Commit 88c9300a authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Merge remote-tracking branch 'remotes/origin/develop' into feature-VarscanIntoShiva

Conflicts:
	public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/varscan/Varscan.scala
parents ae01e8be 6714a0ab
......@@ -15,13 +15,9 @@
*/
package nl.lumc.sasc.biopet.core.extensions
import java.io.{ File, FileOutputStream }
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.utils.rscript.Rscript
import scala.sys.process._
/**
* General rscript extension
*
......
......@@ -184,6 +184,7 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
map.toMap
}
}
object WriteSummary {
/** Retrive checksum from file */
def parseChecksum(checksumFile: File): String = {
......
......@@ -19,6 +19,7 @@ class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar {
@Test
def testDefault: Unit = {
Logging.errors.clear()
make(config :: testReferenceNoIndex :: Nil).referenceFasta()
Logging.checkErrors(true)
......
package nl.lumc.sasc.biopet.core.summary
import java.io.File
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test
/**
* Created by pjvanthof on 14/01/16.
*/
class SummarizableTest extends TestNGSuite with Matchers {
@Test
def testDefaultMerge: Unit = {
val summarizable = new Summarizable {
def summaryFiles: Map[String, File] = ???
def summaryStats: Any = ???
}
intercept[IllegalStateException] {
summarizable.resolveSummaryConflict("1", "1", "key")
}
}
def testOverrideMerge: Unit = {
val summarizable = new Summarizable {
def summaryFiles: Map[String, File] = ???
def summaryStats: Any = ???
override def resolveSummaryConflict(v1: Any, v2: Any, key: String) = v1
}
summarizable.resolveSummaryConflict("1", "1", "key") shouldBe "1"
}
}
package nl.lumc.sasc.biopet.core.summary
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
import nl.lumc.sasc.biopet.core.extensions.Md5sum
import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
import org.broadinstitute.gatk.queue.QScript
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test
import SummaryQScriptTest._
/**
* Created by pjvanthof on 14/01/16.
*/
class SummaryQScriptTest extends TestNGSuite with Matchers {
@Test
def testNoJobs: Unit = {
SummaryQScript.md5sumCache.clear()
val script = makeQscript()
script.addSummaryJobs()
SummaryQScript.md5sumCache shouldBe empty
}
@Test
def testFiles: Unit = {
SummaryQScript.md5sumCache.clear()
val file = new File(s".${File.separator}bla")
val script = makeQscript(files = Map("file" -> file))
script.addSummaryJobs()
SummaryQScript.md5sumCache should not be empty
SummaryQScript.md5sumCache.toMap shouldBe Map(
new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5"))
script.functions.size shouldBe 2
assert(script.functions
.filter(_.isInstanceOf[Md5sum])
.map(_.asInstanceOf[Md5sum])
.exists(_.cmdLine.contains(" || ")))
}
@Test
def testDuplicateFiles: Unit = {
SummaryQScript.md5sumCache.clear()
val file = new File(s".${File.separator}bla")
val script = makeQscript(files = Map("file" -> file, "file2" -> file))
script.addSummaryJobs()
SummaryQScript.md5sumCache should not be empty
SummaryQScript.md5sumCache.toMap shouldBe Map(
new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5"))
script.functions.size shouldBe 2
assert(script.functions
.filter(_.isInstanceOf[Md5sum])
.map(_.asInstanceOf[Md5sum])
.exists(_.cmdLine.contains(" || ")))
}
@Test
def testAddSummarizable: Unit = {
SummaryQScript.md5sumCache.clear()
val file = new File(s".${File.separator}bla")
val script = makeQscript()
script.addSummarizable(makeSummarizable(files = Map("file" -> file, "file2" -> file)), "test")
script.summarizables.size shouldBe 1
script.addSummaryJobs()
SummaryQScript.md5sumCache should not be empty
SummaryQScript.md5sumCache.toMap shouldBe Map(
new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5"))
script.functions.size shouldBe 2
assert(script.functions
.filter(_.isInstanceOf[Md5sum])
.map(_.asInstanceOf[Md5sum])
.exists(_.cmdLine.contains(" || ")))
}
@Test
def testInputFile: Unit = {
SummaryQScript.md5sumCache.clear()
val file = new File(s".${File.separator}bla")
val script = makeQscript()
script.addSummarizable(makeSummarizable(files = Map("file" -> file, "file2" -> file)), "test")
script.summarizables.size shouldBe 1
script.inputFiles :+= InputFile(file, Some("md5sum"))
script.inputFiles :+= InputFile(file, None)
script.addSummaryJobs()
SummaryQScript.md5sumCache should not be empty
SummaryQScript.md5sumCache.toMap shouldBe Map(
new File(s".${File.separator}bla") -> new File(s".${File.separator}bla.md5"))
script.functions.size shouldBe 3
assert(script.functions
.filter(_.isInstanceOf[Md5sum])
.map(_.asInstanceOf[Md5sum])
.exists(_.cmdLine.contains(" || ")))
}
@Test
def testAddQscript: Unit = {
SummaryQScript.md5sumCache.clear()
val script = makeQscript()
script.addSummaryQScript(script)
script.summaryQScripts.head shouldBe script
}
}
object SummaryQScriptTest {
def makeQscript(settings: Map[String, Any] = Map(),
files: Map[String, File] = Map(),
c: Map[String, Any] = Map()) =
new SummaryQScript with QScript {
outputDir = new File(".")
override def globalConfig = new Config(c)
def summarySettings: Map[String, Any] = settings
def summaryFiles: Map[String, File] = files
val tempFile = File.createTempFile("summary", ".json")
tempFile.deleteOnExit()
def summaryFile: File = tempFile
def init(): Unit = ???
def biopetScript(): Unit = ???
def root: Configurable = null
}
def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable {
def summaryFiles: Map[String, File] = files
def summaryStats: Any = stats
}
}
\ No newline at end of file
package nl.lumc.sasc.biopet.core.summary
import java.io.{ PrintWriter, File }
import com.google.common.io.Files
import nl.lumc.sasc.biopet.core._
import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
import nl.lumc.sasc.biopet.utils.summary.Summary
import org.broadinstitute.gatk.queue.function.CommandLineFunction
import org.broadinstitute.gatk.queue.{ QScript, QSettings }
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import WriteSummaryTest._
import org.testng.annotations.Test
import scala.util.matching.Regex
/**
* Created by pjvanthof on 15/01/16.
*/
class WriteSummaryTest extends TestNGSuite with Matchers {
@Test
def testWrongRoot(): Unit = {
intercept[IllegalArgumentException] {
makeWriter(null)
}
}
/** This is a basic summary test, no matter the content this should always be true */
def basicSummaryTest(summary: Summary,
name: String,
sampleId: Option[String] = None,
libId: Option[String] = None): Unit = {
summary.getValue(sampleId, libId, name) should not be None
summary.getValue(sampleId, libId, name, "files", "pipeline").get shouldBe a[Map[_, _]]
summary.getValue(sampleId, libId, name, "settings").get shouldBe a[Map[_, _]]
summary.getValue(sampleId, libId, name, "executables").get shouldBe a[Map[_, _]]
summary.getValue("meta") should not be None
summary.getValue("meta", "pipeline_name") shouldBe Some(name)
summary.getValue("meta", "last_commit_hash") shouldBe Some(nl.lumc.sasc.biopet.LastCommitHash)
summary.getValue("meta", "pipeline_version") shouldBe Some(nl.lumc.sasc.biopet.Version)
summary.getValue("meta", "output_dir") shouldBe Some(new File(".").getAbsolutePath)
summary.getValue("meta", "summary_creation") should not be None
}
def createFakeCheckSum(file: File): Unit = {
file.getParentFile.mkdirs()
val writer = new PrintWriter(file)
writer.println("checksum file")
writer.close()
file.deleteOnExit()
}
@Test
def testEmpty(): Unit = {
val qscript = makeQscript(name = "test")
val writer = makeWriter(qscript)
writer.freezeFieldValues()
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
}
@Test
def testMergeQscript(): Unit = {
val qscript = makeQscript(name = "test")
val qscript2 = makeQscript(name = "test2")
qscript.addSummaryQScript(qscript2)
val summaryWriter = new PrintWriter(qscript2.summaryFile)
summaryWriter.println("""{ "test2": "value" }""")
summaryWriter.close()
val writer = makeWriter(qscript)
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test2") shouldBe Some("value")
}
@Test
def testSingleJob(): Unit = {
val qscript = makeQscript("test")
val writer = makeWriter(qscript)
val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
def testSingleJavaJob(): Unit = {
val qscript = makeQscript("test")
val writer = makeWriter(qscript)
val summarizable = makeJavaCommand(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.add(summarizable)
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
}
@Test
def testVersion(): Unit = {
val qscript = makeQscript("test")
val writer = makeWriter(qscript)
val summarizable = makeVersionSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.add(summarizable)
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
}
@Test
def testSampleLibrary(): Unit = {
val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName"))
val writer = makeWriter(qscript)
val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
def testSample(): Unit = {
val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"))
val writer = makeWriter(qscript)
val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
def testMultisampleQscript(): Unit = {
val qscript = makeMultisampleQscript("test", multisampleConfig)
val writer = makeWriter(qscript)
val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
qscript.addSummarizable(summarizable, "tool_1")
qscript.addSummaryJobs()
createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
writer.freezeFieldValues()
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
}
}
object WriteSummaryTest {
def makeWriter(root: Configurable, c: Map[String, Any] = Map()) = new WriteSummary(root) {
override def globalConfig = new Config(c)
override def outputs = Seq()
override def inputs = Seq()
qSettings = new QSettings {
jobName = "test"
jobTempDir = Files.createTempDir()
jobTempDir.deleteOnExit()
jobPriority = Some(1)
}
override def absoluteCommandDirectory() {}
}
def makeQscript(name: String,
settings: Map[String, Any] = Map(),
files: Map[String, File] = Map(),
c: Map[String, Any] = Map()) =
new SummaryQScript with QScript {
summaryName = name
outputDir = new File(".").getAbsoluteFile
override def globalConfig = new Config(c)
def summarySettings: Map[String, Any] = settings
def summaryFiles: Map[String, File] = files
val tempFile = File.createTempFile("summary", ".json")
tempFile.deleteOnExit()
def summaryFile: File = tempFile
def init(): Unit = {}
def biopetScript(): Unit = {}
def root: Configurable = null
}
def makeSampleLibraryQscript(name: String,
settings: Map[String, Any] = Map(),
files: Map[String, File] = Map(),
c: Map[String, Any] = Map(),
s: Option[String] = None,
l: Option[String] = None) =
new SummaryQScript with QScript with SampleLibraryTag {
sampleId = s
libId = l
summaryName = "test"
outputDir = new File(".").getAbsoluteFile
override def globalConfig = new Config(c)
def summarySettings: Map[String, Any] = settings
def summaryFiles: Map[String, File] = files
val tempFile = File.createTempFile("summary", ".json")
tempFile.deleteOnExit()
def summaryFile: File = tempFile
def init(): Unit = {}
def biopetScript(): Unit = {}
def root: Configurable = null
}
def makeMultisampleQscript(name: String,
c: Map[String, Any],
settings: Map[String, Any] = Map(),
files: Map[String, File] = Map()) =
new MultiSampleQScript with QScript {
summaryName = "test"
outputDir = new File(".").getAbsoluteFile
override def globalConfig = new Config(c)
def summarySettings: Map[String, Any] = settings
def summaryFiles: Map[String, File] = files
val tempFile = File.createTempFile("summary", ".json")
tempFile.deleteOnExit()
def summaryFile: File = tempFile
def init(): Unit = {}
def biopetScript(): Unit = {}
def root: Configurable = null
class Sample(id: String) extends AbstractSample(id) {
class Library(id: String) extends AbstractLibrary(id) {
protected def addJobs(): Unit = {}
def summaryFiles: Map[String, File] = files
def summaryStats: Any = Map()
}
def makeLibrary(id: String): Library = new Library(id)
protected def addJobs(): Unit = {}
def summaryFiles: Map[String, File] = files
def summaryStats: Any = Map()
}
def makeSample(id: String): Sample = new Sample(id)
def addMultiSampleJobs(): Unit = {}
}
val multisampleConfig = Map("samples" -> Map("sampleName" -> Map("libraries" -> Map("libName" -> Map()))))
def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable {
def summaryFiles: Map[String, File] = files
def summaryStats: Any = stats
}
def makeJavaCommand(files: Map[String, File] = Map(),
stats: Map[String, Any] = Map(),
c: Map[String, Any] = Map()) = new BiopetJavaCommandLineFunction with Summarizable with Version {
override def globalConfig = new Config(c)
override def configName = "java_command"
def root: Configurable = null
def summaryStats: Map[String, Any] = stats
def summaryFiles: Map[String, File] = files
def versionCommand: String = "echo test version"
def versionRegex: Regex = """(.*)""".r
override def getVersion = Some("test version")
override def outputs = Seq()
override def inputs = Seq()
qSettings = new QSettings {
jobName = "test"
jobTempDir = Files.createTempDir()
jobTempDir.deleteOnExit()
jobPriority = Some(1)
}
override def absoluteCommandDirectory() {}
}
def makeVersionSummarizable(files: Map[String, File] = Map(),
stats: Map[String, Any] = Map(),
c: Map[String, Any] = Map()) =
new CommandLineFunction with Configurable with Summarizable with Version {
override def globalConfig = new Config(c)
override def configName = "version_command"
def root: Configurable = null
def summaryFiles: Map[String, File] = files
def summaryStats: Any = stats
def versionCommand: String = "echo test version"
def versionRegex: Regex = """(.*)""".r
override def getVersion = Some("test version")
def commandLine: String = ""
override def outputs = Seq()
override def inputs = Seq()
qSettings = new QSettings {
jobName = "test"
jobTempDir = Files.createTempDir()
jobTempDir.deleteOnExit()
jobPriority = Some(1)
}
override def absoluteCommandDirectory() {}
}
}
\ No newline at end of file
......@@ -26,18 +26,19 @@ import scala.collection.mutable
import scala.io.Source
/**
* Extension for cutadept
* Based on version 1.5
* Extension for cutadapt
* Started with version 1.5
* Updated to version 1.9 (18-01-2016 by wyleung)