Commit 166530dc authored by Peter van 't Hof's avatar Peter van 't Hof

Remove old summary code and fixed dependencies for report

parent b1d39961
......@@ -43,13 +43,6 @@ class BamMetrics(val parent: Configurable) extends QScript
override def defaults = Map("bedtoolscoverage" -> Map("sorted" -> true))
/** return location of summary file */
def summaryFile = (sampleId, libId) match {
case (Some(s), Some(l)) => new File(outputDir, s + "-" + l + ".BamMetrics.summary.json")
case (Some(s), _) => new File(outputDir, s + ".BamMetrics.summary.json")
case _ => new File(outputDir, "BamMetrics.summary.json")
}
/** returns files to store in summary */
def summaryFiles = Map("reference" -> referenceFasta(),
"input_bam" -> inputBam) ++
......
......@@ -52,8 +52,6 @@ class Basty(val parent: Configurable) extends QScript with MultiSampleQScript {
lazy val shiva = new Shiva(qscript)
def summaryFile: File = new File(outputDir, "Basty.summary.json")
def summaryFiles: Map[String, File] = Map()
def summarySettings: Map[String, Any] = Map("boot_runs" -> numBoot)
......
......@@ -103,7 +103,6 @@ class BastyTest extends TestNGSuite with Matchers {
pipeline.summarySettings.get("boot_runs") shouldBe Some(bootRuns.getOrElse(100))
pipeline.summaryFile shouldBe new File(BastyTest.outputDir, "Basty.summary.json")
pipeline.summaryFiles shouldBe Map()
pipeline.samples foreach {
......
......@@ -90,6 +90,10 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
logger.info("Write report is skipped because sample flag is used")
case _ => reportClass.foreach { report =>
for (f <- functions) f match {
case w: WriteSummary => report.deps :+= w.jobOutputFile
case _ =>
}
add(report)
}
}
......
......@@ -50,9 +50,6 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File]
/** Name of summary output file */
def summaryFile: File
def summaryDbFile: File = root match {
case s: SummaryQScript => new File(s.outputDir, s"${s.summaryName}.summary.db")
case _ => throw new IllegalStateException("Root should be a SummaryQScript")
......
......@@ -44,9 +44,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
@Input(doc = "deps", required = false)
var deps: List[File] = Nil
@Output(doc = "Summary output", required = true)
var out: File = qscript.summaryFile
var md5sum: Boolean = config("summary_md5", default = true)
//TODO: add more checksums types
......@@ -77,8 +74,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
val pipelineId = Await.result(db.createPipeline(qscript.summaryName, qscript.summaryRunId), Duration.Inf)
qscript.summarizables.map(x => Await.result(db.createModule(x._1._1, qscript.summaryRunId, pipelineId), Duration.Inf))
for (q <- qscript.summaryQScripts)
deps :+= q.summaryFile
for ((_, l) <- qscript.summarizables; s <- l) {
deps :::= s.summaryDeps
s match {
......@@ -91,7 +86,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
}
}
jobOutputFile = new File(qscript.summaryDbFile.getParentFile, "." + qscript.summaryDbFile.getName.stripSuffix(".db") + ".out")
jobOutputFile = new File(qscript.outputDir, s".${qscript.summaryName}.summary.out")
}
def createFile(db: SummaryDb, runId: Int, pipelineId: Int, moduleId: Option[Int], sampleId: Option[Int], libId: Option[Int], key: String, file: File, outputDir: File) = {
......@@ -174,100 +169,6 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
Some(db.createOrUpdateExecutable(qscript.summaryRunId, f.configNamespace, f.getVersion))
case _ => None
}).flatten.foreach(Await.ready(_, Duration.Inf))
///////////////// OLD //////////////////
for (((name, sampleId, libraryId), summarizables) <- qscript.summarizables; summarizable <- summarizables) {
summarizable.addToQscriptSummary(qscript, name)
}
val pipelineMap = {
val files = parseFiles(qscript.summaryFiles)
val settings = qscript.summarySettings
val executables: Map[String, Any] = {
def fetchVersion(f: QFunction): Option[(String, Any)] = {
f match {
case f: BiopetJavaCommandLineFunction with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None),
"java_md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"java_version" -> f.getJavaVersion,
"jar_path" -> f.jarFile))
case f: BiopetCommandLineFunction with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None),
"md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"path" -> f.executable))
case f: Configurable with Version =>
Some(f.configNamespace -> Map("version" -> f.getVersion.getOrElse(None)))
case _ => None
}
}
(
qscript.functions.flatMap(fetchVersion) ++
qscript.functions
.flatMap {
case f: BiopetCommandLineFunction => f.pipesJobs
case _ => Nil
}.flatMap(fetchVersion(_))
).toMap
}
val map = Map(qscript.summaryName -> Map(
"settings" -> settings,
"files" -> Map("pipeline" -> files),
"executables" -> executables)
)
qscript match {
case tag: SampleLibraryTag => prefixSampleLibrary(map, tag.sampleId, tag.libId)
case q: MultiSampleQScript =>
ConfigUtils.mergeMaps(
Map("samples" -> q.samples.map {
case (sampleName, sample) =>
sampleName -> Map(
qscript.summaryName -> Map(
"settings" -> sample.summarySettings,
"tags" -> sample.sampleTags),
"libraries" -> sample.libraries.map {
case (libName, lib) =>
libName -> Map(
qscript.summaryName -> Map(
"settings" -> lib.summarySettings,
"tags" -> lib.libTags)
)
}
)
}), map)
case _ => map
}
}
val jobsMap = (for (
((name, sampleId, libraryId), summarizables) <- qscript.summarizables;
summarizable <- summarizables
) yield {
val map = Map(qscript.summaryName -> parseSummarizable(summarizable, name))
(prefixSampleLibrary(map, sampleId, libraryId),
(v1: Any, v2: Any, key: String) => summarizable.resolveSummaryConflict(v1, v2, key))
}).foldRight(pipelineMap)((a, b) => ConfigUtils.mergeMaps(a._1, b, a._2))
val combinedMap = //(for (qscript <- qscript.summaryQScripts) yield {
// ConfigUtils.fileToConfigMap(qscript.summaryFile)
// }).foldRight(jobsMap)((a, b) => ConfigUtils.mergeMaps(a, b)) ++
jobsMap ++ Map("meta" -> Map(
"last_commit_hash" -> LastCommitHash,
"pipeline_version" -> nl.lumc.sasc.biopet.Version,
"pipeline_name" -> qscript.summaryName,
"output_dir" -> qscript.outputDir,
"run_name" -> config("run_name", default = qSettings.runName).asString,
"summary_creation" -> System.currentTimeMillis()
))
val writer = new PrintWriter(out)
writer.println(ConfigUtils.mapToJson(combinedMap).nospaces)
writer.close()
///////////////// OLD //////////////////
}
def prefixSampleLibrary(map: Map[String, Any], sampleId: Option[String], libraryId: Option[String]): Map[String, Any] = {
......
......@@ -75,8 +75,8 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
}
@Test
......@@ -91,9 +91,11 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test2") shouldBe Some("value")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test2") shouldBe Some("value")
}
@Test
......@@ -107,10 +109,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -125,11 +129,13 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
// summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
}
@Test
......@@ -144,11 +150,13 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.freezeFieldValues()
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
// summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
}
@Test
......@@ -163,10 +171,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
// summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -181,10 +191,12 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
// summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
}
@Test
......@@ -199,12 +211,14 @@ class WriteSummaryTest extends TestNGSuite with Matchers {
writer.deps shouldBe empty
writer.run()
val summary = new Summary(writer.out)
basicSummaryTest(summary, "test")
summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
throw new Exception("Fix me")
summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
// val summary = new Summary(writer.out)
// basicSummaryTest(summary, "test")
// summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
// summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
//
// summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
}
}
......
......@@ -54,8 +54,6 @@ class Carp(val parent: Configurable) extends QScript with MultisampleMappingTrai
"macs2callpeak" -> Map("fileformat" -> "")
)
def summaryFile = new File(outputDir, "Carp.summary.json")
override def makeSample(id: String) = new Sample(id)
class Sample(sampleId: String) extends super.Sample(sampleId) {
......
......@@ -27,9 +27,6 @@ import org.broadinstitute.gatk.queue.QScript
class BiopetPipeline(val parent: Configurable) extends QScript with SummaryQScript {
def this() = this(null)
/** Only required when using [[SummaryQScript]] */
def summaryFile = new File(outputDir, "magpie.summary.json")
/** Only required when using [[SummaryQScript]] */
def summaryFiles: Map[String, File] = Map()
......
......@@ -23,9 +23,6 @@ import org.broadinstitute.gatk.queue.QScript
class HelloPipeline(val parent: Configurable) extends QScript with SummaryQScript {
def this() = this(null)
/** Only required when using [[SummaryQScript]] */
def summaryFile = new File(outputDir, "hello.summary.json")
/** Only required when using [[SummaryQScript]] */
def summaryFiles: Map[String, File] = Map()
......
......@@ -37,8 +37,6 @@ class MultisamplePipeline(val parent: Configurable) extends QScript with MultiSa
// this code will be executed after all code of all samples is executed
}
def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
//TODO: Add summary
def summaryFiles: Map[String, File] = Map()
......
......@@ -41,9 +41,6 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
/** Make a final fastq files, by default only when flexiprep is the main pipeline */
var keepQcFastqFiles: Boolean = config("keepQcFastqFiles", default = parent == null)
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("x") + "-" + libId.getOrElse("x") + ".qc.summary.json")
/** Returns files to store in summary */
def summaryFiles: Map[String, File] = {
Map("input_R1" -> inputR1, "output_R1" -> fastqR1Qc) ++
......
......@@ -95,7 +95,4 @@ class CombineReads(val parent: Configurable) extends QScript with SummaryQScript
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map()
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "combine_reads.summary.json")
}
......@@ -45,9 +45,6 @@ class Gears(val parent: Configurable) extends QScript with MultiSampleQScript {
def init(): Unit = {
}
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "gears.summary.json")
/** Pipeline itself */
def biopetScript(): Unit = {
addSamplesJobs()
......
......@@ -77,9 +77,6 @@ class GearsCentrifuge(val parent: Configurable) extends QScript with SummaryQScr
addSummarizable(krakenReportJSON, s"${name}_report")
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".centrifuge.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map()
......
......@@ -100,9 +100,6 @@ class GearsKraken(val parent: Configurable) extends QScript with SummaryQScript
addSummaryJobs()
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".kraken.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map()
......
......@@ -77,9 +77,6 @@ class GearsQiimeClosed(val parent: Configurable) extends QScript with SummaryQSc
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map("otu_table" -> otuTable, "otu_map" -> otuMap)
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "summary.closed_reference.json")
val downSample: Option[Double] = config("downsample")
def addDownsample(input: File, output: File): File = {
......
......@@ -70,9 +70,6 @@ class GearsQiimeOpen(val parent: Configurable) extends QScript with SummaryQScri
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = Map("otu_table" -> otuTable, "otu_map" -> otuMap)
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "summary.open_reference.json")
val downSample: Option[Double] = config("downsample")
def addDownsample(input: File, output: File): File = {
......
......@@ -180,9 +180,6 @@ class GearsSingle(val parent: Configurable) extends QScript with SummaryQScript
addSummaryJobs()
}
/** Location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("sampleName_unknown") + ".gears.summary.json")
/** Pipeline settings shown in the summary file */
def summarySettings: Map[String, Any] = Map(
"skip_flexiprep" -> skipFlexiprep,
......
......@@ -152,9 +152,6 @@ class Gentrap(val parent: Configurable) extends QScript
Some(pipeline)
} else None
/** Output summary file */
def summaryFile: File = new File(outputDir, "gentrap.summary.json")
/** Files that will be listed in the summary file */
override def summaryFiles: Map[String, File] = super.summaryFiles ++ Map(
"annotation_refflat" -> annotationRefFlat()
......
......@@ -83,7 +83,4 @@ trait Measurement extends SummaryQScript with Reference { qscript: QScript =>
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = extraSummaryFiles ++ bamFiles.map { case (id, file) => s"input_bam_$id" -> file }
/** Name of summary output file */
def summaryFile: File = new File(outputDir, s"$name.summary.json")
}
......@@ -95,9 +95,6 @@ class Kopisu(val parent: Configurable) extends QScript with SummaryQScript with
/** File to put in the summary for thie pipeline */
def summaryFiles: Map[String, File] = inputBams.map(x => s"inputbam_${x._1}" -> x._2)
/** Name of summary output file */
def summaryFile: File = new File(outputDir, "kopisu.summary.json")
}
object Kopisu extends PipelineCommand
......@@ -30,9 +30,6 @@ trait CnvMethod extends QScript with SummaryQScript with Reference {
var inputBams: Map[String, File] = Map.empty
/** Name of summary output file */
def summaryFile: File = new File(outputDir, s"$name.summary.json")
/** Must return a map with used settings for this pipeline */
def summarySettings: Map[String, Any] = Map()
......
......@@ -101,9 +101,6 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
new File(outputDir, outputName + ".bam")
} else new File(outputDir, outputName + ".dedup.bam")
/** location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("x") + "-" + libId.getOrElse("x") + ".summary.json")
override def defaults: Map[String, Any] = Map(
"gsnap" -> Map("batch" -> 4),
"star" -> Map("outsamunmapped" -> "Within")
......
......@@ -296,8 +296,6 @@ trait MultisampleMappingTrait extends MultiSampleQScript
/** This class is the default implementation that can be used on the command line */
class MultisampleMapping(val parent: Configurable) extends QScript with MultisampleMappingTrait {
def this() = this(null)
def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
}
object MultisampleMapping extends PipelineCommand {
......
......@@ -52,8 +52,6 @@ class Sage(val parent: Configurable) extends QScript with MultiSampleQScript {
), "strandSensitive" -> true
)
def summaryFile: File = new File(outputDir, "Sage.summary.json")
def summaryFiles: Map[String, File] = Map()
def summarySettings: Map[String, Any] = Map()
......
......@@ -81,7 +81,6 @@ class SageTest extends TestNGSuite with Matchers {
val numberLibs = (if (sample1) 1 else 0) + (if (sample2) 2 else 0)
val numberSamples = (if (sample1) 1 else 0) + (if (sample2) 1 else 0)
pipeline.summaryFile shouldBe new File(SageTest.outputDir, "Sage.summary.json")
pipeline.summaryFiles shouldBe Map()
pipeline.summarySettings shouldBe Map()
......
......@@ -208,9 +208,6 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
}
}
/** Location of summary file */
def summaryFile = new File(outputDir, "Shiva.summary.json")
/** Settings of pipeline for summary */
override def summarySettings = super.summarySettings ++ Map(
"annotation" -> annotation.isDefined,
......
......@@ -97,9 +97,6 @@ class ShivaSvCalling(val parent: Configurable) extends QScript with SummaryQScri
/** Will generate all available variantcallers */
protected def callersList: List[SvCaller] = List(new Breakdancer(this), new Clever(this), new Delly(this), new Pindel(this))
/** Location of summary file */
def summaryFile = new File(outputDir, "ShivaSvCalling.summary.json")
/** Settings for the summary */
def summarySettings = Map("sv_callers" -> configCallers.toList)
......
......@@ -163,9 +163,6 @@ class ShivaVariantcalling(val parent: Configurable) extends QScript
}
}
/** Location of summary file */
def summaryFile = new File(outputDir, "ShivaVariantcalling.summary.json")
/** Settings for the summary */
def summarySettings = Map(
"variantcallers" -> configCallers.toList,
......
......@@ -108,8 +108,6 @@ class TinyCap(val parent: Configurable) extends QScript
}
}
override def summaryFile = new File(outputDir, "tinycap.summary.json")
override def summaryFiles: Map[String, File] = super.summaryFiles ++ Map(
"annotation_refflat" -> annotationRefFlat(),
"annotationGtf" -> annotationGtf,
......
......@@ -284,8 +284,6 @@ class Toucan(val parent: Configurable) extends QScript with BiopetQScript with S
finalLn.output
}
def summaryFile = new File(outputDir, "Toucan.summary.json")
def summaryFiles = Map("input_vcf" -> inputVcf, "outputVcf" -> outputVcf)
def summarySettings = Map()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment