Skip to content

GitLab

Commit 166530dc authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Remove old summary code and fixed dependencies for report

parent b1d39961
Hide whitespace changes
Inline Side-by-side
bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BamMetrics.scala
View file @ 166530dc
......@@ -43,13 +43,6 @@ class BamMetrics(val parent: Configurable) extends QScript
override def defaults = Map("bedtoolscoverage" -> Map("sorted" -> true))
/** return location of summary file */
def summaryFile = (sampleId, libId) match {
case (Some(s), Some(l)) => new File(outputDir, s + "-" + l + ".BamMetrics.summary.json")
case (Some(s), _) => new File(outputDir, s + ".BamMetrics.summary.json")
case _ => new File(outputDir, "BamMetrics.summary.json")
}
/** returns files to store in summary */
def summaryFiles = Map("reference" -> referenceFasta(),
"input_bam" -> inputBam) ++
......
basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/Basty.scala
View file @ 166530dc
......@@ -52,8 +52,6 @@ class Basty(val parent: Configurable) extends QScript with MultiSampleQScript {
lazy val shiva = new Shiva(qscript)
def summaryFile: File = new File(outputDir, "Basty.summary.json")
def summaryFiles: Map[String, File] = Map()
def summarySettings: Map[String, Any] = Map("boot_runs" -> numBoot)
......
basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala
View file @ 166530dc
......@@ -103,7 +103,6 @@ class BastyTest extends TestNGSuite with Matchers {
pipeline.summarySettings.get("boot_runs") shouldBe Some(bootRuns.getOrElse(100))
pipeline.summaryFile shouldBe new File(BastyTest.outputDir, "Basty.summary.json")
pipeline.summaryFiles shouldBe Map()
pipeline.samples foreach {
......
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala
View file @ 166530dc
......@@ -90,6 +90,10 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
logger.info("Write report is skipped because sample flag is used")
case _ => reportClass.foreach { report =>
for (f <- functions) f match {
case w: WriteSummary => report.deps :+= w.jobOutputFile
case _ =>
}
add(report)
}
}
......
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala
View file @ 166530dc
......@@ -50,9 +50,6 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File]
/** Name of summary output file */
def summaryFile: File
def summaryDbFile: File = root match {
case s: SummaryQScript => new File(s.outputDir, s"${s.summaryName}.summary.db")
case _ => throw new IllegalStateException("Root should be a SummaryQScript")
......
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/WriteSummary.scala
View file @ 166530dc
......@@ -44,9 +44,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
@Input(doc = "deps", required = false)
var deps: List[File] = Nil
@Output(doc = "Summary output", required = true)
var out: File = qscript.summaryFile
var md5sum: Boolean = config("summary_md5", default = true)
//TODO: add more checksums types
......@@ -77,8 +74,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
val pipelineId = Await.result(db.createPipeline(qscript.summaryName, qscript.summaryRunId), Duration.Inf)
qscript.summarizables.map(x => Await.result(db.createModule(x._1._1, qscript.summaryRunId, pipelineId), Duration.Inf))
for (q <- qscript.summaryQScripts)
deps :+= q.summaryFile
for ((_, l) <- qscript.summarizables; s <- l) {
deps :::= s.summaryDeps
s match {
......@@ -91,7 +86,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
}
}
jobOutputFile = new File(qscript.summaryDbFile.getParentFile, "." + qscript.summaryDbFile.getName.stripSuffix(".db") + ".out")
jobOutputFile = new File(qscript.outputDir, s".${qscript.summaryName}.summary.out")
}
def createFile(db: SummaryDb, runId: Int, pipelineId: Int, moduleId: Option[Int], sampleId: Option[Int], libId: Option[Int], key: String, file: File, outputDir: File) = {
......@@ -174,100 +169,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
Some(db.createOrUpdateExecutable(qscript.summaryRunId, f.configNamespace, f.getVersion))
case _ => None
}).flatten.foreach(Await.ready(_, Duration.Inf))
///////////////// OLD //////////////////
for (((name, sampleId, libraryId), summarizables) <- qscript.summarizables; summarizable <- summarizables) {
summarizable.addToQscriptSummary(qscript, name)
}
val pipelineMap = {
val files = parseFiles(qscript.summaryFiles)
val settings = qscript.summarySettings
val executables: Map[String, Any] = {
def fetchVersion(f: QFunction): Option[(String, Any)] = {
f match {
case f: BiopetJavaCommandLineFunction with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None),
"java_md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"java_version" -> f.getJavaVersion,
"jar_path" -> f.jarFile))
case f: BiopetCommandLineFunction with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None),
"md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"path" -> f.executable))
case f: Configurable with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None)))
case _ => None
}
}
(
qscript.functions.flatMap(fetchVersion) ++
qscript.functions
.flatMap {
case f: BiopetCommandLineFunction => f.pipesJobs
case _ => Nil
}.flatMap(fetchVersion(_))
).toMap
}
val map = Map(qscript.summaryName -> Map(
"settings" -> settings,
"files" -> Map("pipeline" -> files),
"executables" -> executables)
)
qscript match {
case tag: SampleLibraryTag => prefixSampleLibrary(map, tag.sampleId, tag.libId)
case q: MultiSampleQScript =>
ConfigUtils.mergeMaps(
Map("samples" -> q.samples.map {
case (sampleName, sample) =>
sampleName -> Map(
qscript.summaryName -> Map(
"settings" -> sample.summarySettings,
"tags" -> sample.sampleTags),
"libraries" -> sample.libraries.map {
case (libName, lib) =>
libName -> Map(
qscript.summaryName -> Map(
"settings" -> lib.summarySettings,
"tags" -> lib.libTags)
)
}
)
}), map)
case _ => map
}
}
val jobsMap = (for (
((name, sampleId, libraryId), summarizables) <- qscript.summarizables;
summarizable <- summarizables
) yield {
val map = Map(qscript.summaryName -> parseSummarizable(summarizable, name))
(prefixSampleLibrary(map, sampleId, libraryId),
(v1: Any, v2: Any, key: String) => summarizable.resolveSummaryConflict(v1, v2, key))
}).foldRight(pipelineMap)((a, b) => ConfigUtils.mergeMaps(a._1, b, a._2))
val combinedMap = //(for (qscript <- qscript.summaryQScripts) yield {
// ConfigUtils.fileToConfigMap(qscript.summaryFile)
// }).foldRight(jobsMap)((a, b) => ConfigUtils.mergeMaps(a, b)) ++
jobsMap ++ Map("meta" -> Map(
"last_commit_hash" -> LastCommitHash,
"pipeline_version" -> nl.lumc.sasc.biopet.Version,
"pipeline_name" -> qscript.summaryName,
"output_dir" -> qscript.outputDir,
"run_name" -> config("run_name", default = qSettings.runName).asString,
"summary_creation" -> System.currentTimeMillis()
))
val writer = new PrintWriter(out)
writer.println(ConfigUtils.mapToJson(combinedMap).nospaces)
writer.close()
///////////////// OLD //////////////////
}
def prefixSampleLibrary(map: Map[String, Any], sampleId: Option[String], libraryId: Option[String]): Map[String, Any] = {
......
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala
View file @ 166530dc
......@@ -75,8 +75,8 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
}
@Test
......@@ -91,9 +91,11 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test2") shouldBe Some("value")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test2") shouldBe Some("value")
}
@Test
......@@ -107,10 +109,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -125,11 +129,13 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
// summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
}
@Test
......@@ -144,11 +150,13 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
// summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
}
@Test
......@@ -163,10 +171,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
// summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -181,10 +191,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
// summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -199,12 +211,14 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
//
// summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
}
}
......
carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala
View file @ 166530dc
......@@ -54,8 +54,6 @@ class Carp(val parent: Configurable) extends QScript with MultisampleMappingTrai
"macs2callpeak" -> Map("fileformat" -> "")
)
def summaryFile = new File(outputDir, "Carp.summary.json")
override def makeSample(id: String) = new Sample(id)
class Sample(sampleId: String) extends super.Sample(sampleId) {
......
external-example/src/main/scala/org/example/group/pipelines/BiopetPipeline.scala
View file @ 166530dc
......@@ -27,9 +27,6 @@ import org.broadinstitute.gatk.queue.QScript
class BiopetPipeline(val parent: Configurable) extends QScript with SummaryQScript {
def this() = this(null)
/** Only required when using [[SummaryQScript]] */
def summaryFile = new File(outputDir, "magpie.summary.json")
/** Only required when using [[SummaryQScript]] */
def summaryFiles: Map[String, File] = Map()
......
external-example/src/main/scala/org/example/group/pipelines/HelloPipeline.scala
View file @ 166530dc
......@@ -23,9 +23,6 @@ import org.broadinstitute.gatk.queue.QScript
class HelloPipeline(val parent: Configurable) extends QScript with SummaryQScript {
def this() = this(null)
/** Only required when using [[SummaryQScript]] */
def summaryFile = new File(outputDir, "hello.summary.json")
/** Only required when using [[SummaryQScript]] */
def summaryFiles: Map[String, File] = Map()
......
external-example/src/main/scala/org/example/group/pipelines/MultisamplePipeline.scala
View file @ 166530dc
......@@ -37,8 +37,6 @@ class MultisamplePipeline(val parent: Configurable) extends QScript with MultiSa
// this code will be executed after all code of all samples is executed
}
def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
//TODO: Add summary
def summaryFiles: Map[String, File] = Map()
......
flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala
View file @ 166530dc
......@@ -41,9 +41,6 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
/** Make a final fastq files, by default only when flexiprep is the main pipeline */
var keepQcFastqFiles: Boolean = config("keepQcFastqFiles", default = parent == null)
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("x") + "-" + libId.getOrElse("x") + ".qc.summary.json")
/** Returns files to store in summary */
def summaryFiles: Map[String, File] = {
Map("input_R1" -> inputR1, "output_R1" -> fastqR1Qc) ++
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/CombineReads.scala
View file @ 166530dc
......@@ -95,7 +95,4 @@ class CombineReads(val parent: Configurable) extends QScript with SummaryQScript
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map()
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "combine_reads.summary.json")
}
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/Gears.scala
View file @ 166530dc
......@@ -45,9 +45,6 @@ class Gears(val parent: Configurable) extends QScript with MultiSampleQScript {
def init(): Unit = {
}
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "gears.summary.json")
/** Pipeline itself */
def biopetScript(): Unit = {
addSamplesJobs()
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsCentrifuge.scala
View file @ 166530dc
......@@ -77,9 +77,6 @@ class GearsCentrifuge(val parent: Configurable) extends QScript with SummaryQScr
addSummarizable(krakenReportJSON, s"${name}_report")
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".centrifuge.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map()
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsKraken.scala
View file @ 166530dc
......@@ -100,9 +100,6 @@ class GearsKraken(val parent: Configurable) extends QScript with SummaryQScript
addSummaryJobs()
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".kraken.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map()
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsQiimeClosed.scala
View file @ 166530dc
......@@ -77,9 +77,6 @@ class GearsQiimeClosed(val parent: Configurable) extends QScript with SummaryQSc
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map("otu_table" -> otuTable, "otu_map" -> otuMap)
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "summary.closed_reference.json")
val downSample: Option[Double] = config("downsample")
def addDownsample(input: File, output: File): File = {
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsQiimeOpen.scala
View file @ 166530dc
......@@ -70,9 +70,6 @@ class GearsQiimeOpen(val parent: Configurable) extends QScript with SummaryQScri
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map("otu_table" -> otuTable, "otu_map" -> otuMap)
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "summary.open_reference.json")
val downSample: Option[Double] = config("downsample")
def addDownsample(input: File, output: File): File = {
......
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsSingle.scala
View file @ 166530dc
......@@ -180,9 +180,6 @@ class GearsSingle(val parent: Configurable) extends QScript with SummaryQScript
addSummaryJobs()
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".gears.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map(
"skip_flexiprep" -> skipFlexiprep,
......
gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala
View file @ 166530dc
......@@ -152,9 +152,6 @@ class Gentrap(val parent: Configurable) extends QScript
Some(pipeline)
} else None
/** Output summary file */
def summaryFile: File = new File(outputDir, "gentrap.summary.json")
/** Files that will be listed in the summary file */
override def summaryFiles: Map[String, File] = super.summaryFiles ++ Map(
"annotation_refflat" -> annotationRefFlat()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment