Commit cc3369d5 authored by Wai Yi Leung's avatar Wai Yi Leung

Merge branch 'feature-report' into 'develop'

Feature report

second phase for reporting

Adding report to all general pipelines

See merge request !184
parents 290bc3da 8282c888
#{ //TODO: Need content }#
#import(nl.lumc.sasc.biopet.core.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
<%@ var summary: Summary %>
<%@ var rootPath: String %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
Todo
\ No newline at end of file
<table class="table">
<tbody>
<tr><th>Pipeline</th><td>BamMetrics</td></tr>
<tr><th>Version</th><td>${summary.getValue("meta", "pipeline_version")}</td></tr>
<tr><th>Last commit hash</th><td>${summary.getValue("meta", "last_commit_hash")}</td></tr>
<tr><th>Output directory</th><td>${summary.getValue("meta", "output_dir")}</td></tr>
<tr><th>Sample ID</th><td>${sampleId}</td></tr>
#if (libId.isDefined) <tr><th>Library ID</th><td>${libId}</td></tr> #end
</tbody>
</table>
<br/>
<div class="row">
<div class="col-md-1"></div>
<div class="col-md-6">
<p>
In this web document you can find your <em>BamMetrics</em> pipeline report.
Different categories of data can be found in the left-side menu.
Statistics per sample and library can be accessed through the top-level menu.
Futhermore, you can view all versions of software tools used by selecting <em>Versions</em> from the top menu.
</p>
<p>
<small>Brought to you by <a href="https://sasc.lumc.nl">SASC</a> and <a href="https://www.lumc.nl/org/klinische-genetica/">KG</a>, LUMC. </small>
</p>
</div>
</div>
\ No newline at end of file
#import(nl.lumc.sasc.biopet.core.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(java.io.File)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
<%@ var metricsTag: String = "bammetrics" %>
<table class="table sortable-theme-bootstrap">
<thead><tr>
<th>Path</th>
<th>MD5</th>
</tr></thead>
<tbody>
<tr>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "path")}</td>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "md5")}</td>
</tr>
</tbody>
</table>
\ No newline at end of file
......@@ -10,17 +10,17 @@
<%@ var metricsTag: String = "bammetrics" %>
<%@ var target: String %>
#{
val originalPlot = new File(summary.getLibraryValue(sampleId, libId, metricsTag, "files", target + "_cov_stats", "plot", "path")
val originalPlot = new File(summary.getValue(sampleId, libId, metricsTag, "files", target + "_cov_stats", "plot", "path")
.getOrElse(throw new IllegalArgumentException("No plot found in summary")).toString)
val plot = new File(outputDir, target + "_cov_stats.png")
val values = summary.getLibraryValue(sampleId, libId, metricsTag, "stats", target + "_cov_stats", "coverage", "_all")
val values = summary.getValue(sampleId, libId, metricsTag, "stats", target + "_cov_stats", "coverage", "_all")
.getOrElse(throw new IllegalArgumentException("No plot found in summary")).asInstanceOf[Map[String, Any]]
if (originalPlot.exists()) IoUtils.copyFile(originalPlot, plot)
}#
<img src="${plot}">
<img src="${plot.getName}">
<table class="table">
<thead><tr>
......
......@@ -58,6 +58,17 @@ class BamMetrics(val root: Configurable) extends QScript with SummaryQScript wit
def summarySettings = Map("amplicon_name" -> ampliconBedFile.collect { case x => x.getName.stripSuffix(".bed") },
"roi_name" -> roiBedFiles.map(_.getName.stripSuffix(".bed")))
override def reportClass = {
val bammetricsReport = new BammetricsReport(this)
bammetricsReport.outputDir = new File(outputDir, "report")
bammetricsReport.summaryFile = summaryFile
bammetricsReport.args = if (libId.isDefined) Map(
"sampleId" -> sampleId.getOrElse("."),
"libId" -> libId.getOrElse("."))
else Map("sampleId" -> sampleId.getOrElse("."))
Some(bammetricsReport)
}
/** executed before script */
def init() {
}
......
......@@ -2,24 +2,48 @@ package nl.lumc.sasc.biopet.pipelines.bammetrics
import java.io.{ PrintWriter, File }
import nl.lumc.sasc.biopet.core.report.{ ReportBuilder, ReportPage, ReportSection }
import nl.lumc.sasc.biopet.core.config.Configurable
import nl.lumc.sasc.biopet.core.report.{ ReportBuilderExtension, ReportBuilder, ReportPage, ReportSection }
import nl.lumc.sasc.biopet.core.summary.{ SummaryValue, Summary }
import nl.lumc.sasc.biopet.extensions.rscript.{ XYPlot, StackedBarPlot }
class BammetricsReport(val root: Configurable) extends ReportBuilderExtension {
val builder = BammetricsReport
}
/**
* Created by pjvan_thof on 3/30/15.
*/
object BammetricsReport extends ReportBuilder {
// FIXME: Not yet finished
/** Name of report */
val reportName = "Bam Metrics"
def indexPage = ReportPage(List(), List(), Map())
/** Root page for single BamMetrcis report */
def indexPage = {
val bamMetricsPage = this.bamMetricsPage(summary, sampleId, libId)
ReportPage(bamMetricsPage.subPages ::: List(
"Versions" -> ReportPage(List(), List((
"Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
))), Map()),
"Files" -> ReportPage(List(), List(
"Input fastq files" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bammetricsInputFile.ssp")
), Map())
), List(
"Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
) ::: bamMetricsPage.sections,
Map()
)
}
def bamMetricsPage(summary: Summary, sampleId: Option[String], libId: Option[String]) = {
/** Generates a page with alignment stats */
def bamMetricsPage(summary: Summary,
sampleId: Option[String],
libId: Option[String],
metricsTag: String = "bammetrics") = {
val targets = (
summary.getLibraryValue(sampleId, libId, "bammetrics", "settings", "amplicon_name"),
summary.getLibraryValue(sampleId, libId, "bammetrics", "settings", "roi_name")
summary.getValue(sampleId, libId, "bammetrics", "settings", "amplicon_name"),
summary.getValue(sampleId, libId, "bammetrics", "settings", "roi_name")
) match {
case (Some(amplicon: String), Some(roi: List[_])) => amplicon :: roi.map(_.toString)
case (_, Some(roi: List[_])) => roi.map(_.toString)
......@@ -33,12 +57,21 @@ object BammetricsReport extends ReportBuilder {
Map()))),
List(
"Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"),
"Insert Size" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/insertSize.ssp", Map("showPlot" -> true))
"Insert Size" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/insertSize.ssp", Map("showPlot" -> true)),
"Whole genome coverage" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/wgsHistogram.ssp", Map("showPlot" -> true))
),
Map()
Map("metricsTag" -> metricsTag)
)
}
/**
* Generate a stackbar plot for alignment stats
* @param outputDir OutputDir for the tsv and png file
* @param prefix Prefix of the tsv and png file
* @param summary Summary class
* @param libraryLevel Default false, when set true plot will be based on library stats instead of sample stats
* @param sampleId Default it selects all sampples, when sample is giving it limits to selected sample
*/
def alignmentSummaryPlot(outputDir: File,
prefix: String,
summary: Summary,
......@@ -94,6 +127,14 @@ object BammetricsReport extends ReportBuilder {
plot.runLocal()
}
/**
* Generate a line plot for insertsize
* @param outputDir OutputDir for the tsv and png file
* @param prefix Prefix of the tsv and png file
* @param summary Summary class
* @param libraryLevel Default false, when set true plot will be based on library stats instead of sample stats
* @param sampleId Default it selects all sampples, when sample is giving it limits to selected sample
*/
def insertSizePlot(outputDir: File,
prefix: String,
summary: Summary,
......@@ -174,6 +215,14 @@ object BammetricsReport extends ReportBuilder {
plot.runLocal()
}
/**
* Generate a line plot for wgs coverage
* @param outputDir OutputDir for the tsv and png file
* @param prefix Prefix of the tsv and png file
* @param summary Summary class
* @param libraryLevel Default false, when set true plot will be based on library stats instead of sample stats
* @param sampleId Default it selects all sampples, when sample is giving it limits to selected sample
*/
def wgsHistogramPlot(outputDir: File,
prefix: String,
summary: Summary,
......
......@@ -4,9 +4,9 @@
<%@ var rootPath: String %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var pipeline: String = summary.getValue("meta", "pipeline_name").getOrElse("").toString %>
#{
val pipeline = summary.getValue("meta", "pipeline_name").getOrElse("").toString
val executables = summary.getLibraryValue(sampleId, libId, pipeline, "executables").getOrElse(Map()).asInstanceOf[Map[String, Map[String, Any]]]
val executables = summary.getValue(sampleId, libId, pipeline, "executables").getOrElse(Map()).asInstanceOf[Map[String, Map[String, Any]]]
}#
<table class="table">
......
#import(nl.lumc.sasc.biopet.core.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
<%@ var summary: Summary %>
<%@ var rootPath: String %>
<%@ var pipeline: String %>
#{
val contigs = summary.getValue(pipeline, "settings", "reference", "contigs").get.asInstanceOf[Map[String, Map[String, Any]]]
}#
<table class="table">
<tbody>
<tr><th>Species</th><td>${summary.getValue(pipeline, "settings", "reference", "species")}</td></tr>
<tr><th>Name</th><td>${summary.getValue(pipeline, "settings", "reference", "name")}</td></tr>
<tr><th>File</th><td>${summary.getValue(pipeline, "files", "pipeline", "referenceFasta", "path")}</td></tr>
<tr><th>MD5</th><td>${summary.getValue(pipeline, "files", "pipeline", "referenceFasta", "md5")}</td></tr>
</tbody>
</table>
<br/>
<table class="table sortable-theme-bootstrap" data-sortable>
<thead>
<tr><th>Contig Name</th><th data-sorted="true" data-sorted-direction="descending">Length</th><th>MD5</th></tr>
</thead>
<tbody>
#for (c <- contigs.toList.sortBy(_._2("length").asInstanceOf[Long]).reverse)
<tr><th>${c._1}</th><td>${c._2.get("length")}</td><td>${c._2.get("md5")}</td></tr>
#end
</tbody>
</table>
\ No newline at end of file
......@@ -4,22 +4,28 @@ package nl.lumc.sasc.biopet.core.report
* Created by pjvan_thof on 3/30/15.
*/
trait MultisampleReportBuilder extends ReportBuilder {
/** Method to generate a single sample page */
def samplePage(sampleId: String, args: Map[String, Any]): ReportPage
/** Default list of samples, can be override */
def samplesSections: List[(String, ReportSection)] = {
List(
("Samples", ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
)
}
/** Method to generate a single library page */
def libraryPage(sampleId: String, libraryId: String, args: Map[String, Any]): ReportPage
/** Default list of libraries, can be override */
def libririesSections: List[(String, ReportSection)] = {
List(
("Libraries", ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
)
}
/** Generate the samples page including a single sample page for each sample in the summary */
def generateSamplesPage(args: Map[String, Any]): ReportPage = {
val samplePages = summary.samples
.map(sampleId => (sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId)))))
......@@ -27,6 +33,7 @@ trait MultisampleReportBuilder extends ReportBuilder {
ReportPage(samplePages, samplesSections, args)
}
/** Generate the libraries page for a single sample with a subpage for eacht library */
def generateLibraryPage(args: Map[String, Any]): ReportPage = {
val sampleId = args("sampleId") match {
case Some(x) => x.toString
......
......@@ -2,65 +2,81 @@ package nl.lumc.sasc.biopet.core.report
import java.io._
import nl.lumc.sasc.biopet.core.{ ToolCommandFuntion, BiopetJavaCommandLineFunction, ToolCommand }
import nl.lumc.sasc.biopet.core.{ Logging, ToolCommandFuntion, ToolCommand }
import nl.lumc.sasc.biopet.core.summary.Summary
import org.broadinstitute.gatk.utils.commandline.Input
import org.fusesource.scalate.{ TemplateSource, TemplateEngine }
import nl.lumc.sasc.biopet.utils.IoUtils
import scala.collection.mutable
/**
* Created by pjvan_thof on 3/27/15.
*/
trait ReportBuilderExtension extends ToolCommandFuntion {
/** Report builder object */
val builder: ReportBuilder
@Input(required = true)
var summaryFile: File = _
/** OutputDir for the report */
var outputDir: File = _
/** Arguments that are passed on the commandline */
var args: Map[String, String] = Map()
override val defaultCoreMemory = 3.0
override def beforeGraph: Unit = {
super.beforeGraph
jobOutputFile = new File(outputDir, ".report.log.out")
javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
}
/** Command to generate the report */
override def commandLine: String = {
super.commandLine +
required("--summary", summaryFile) +
required("--outputDir", outputDir) +
args.map(x => required(x._1, x._2)).mkString
args.map(x => required("-a", x._1 + "=" + x._2)).mkString
}
}
trait ReportBuilder extends ToolCommand {
case class Args(summary: File = null, outputDir: File = null, pageArgs: Map[String, String] = Map()) extends AbstractArgs
case class Args(summary: File = null, outputDir: File = null, pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
class OptParser extends AbstractOptParser {
opt[File]('s', "summary") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
opt[File]('s', "summary") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(summary = x)
}
opt[File]('o', "outputDir") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(outputDir = x)
}
opt[Map[String, String]]('a', "args") action { (x, c) =>
opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
c.copy(pageArgs = c.pageArgs ++ x)
}
}
/** summary object internaly */
private var setSummary: Summary = _
/** Retrival of summary, read only */
final def summary = setSummary
/** default args that are passed to all page withing the report */
def pageArgs: Map[String, Any] = Map()
private var done = 0
private var total = 0
private var _sampleId: Option[String] = None
protected def sampleId = _sampleId
private var _libId: Option[String] = None
protected def libId = _libId
/** Main function to for building the report */
def main(args: Array[String]): Unit = {
logger.info("Start")
......@@ -70,6 +86,22 @@ trait ReportBuilder extends ToolCommand {
require(cmdArgs.outputDir.exists(), "Output dir does not exist")
require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")
cmdArgs.pageArgs.get("sampleId") match {
case Some(s: String) => {
cmdArgs.pageArgs += "sampleId" -> Some(s)
_sampleId = Some(s)
}
case _ =>
}
cmdArgs.pageArgs.get("libId") match {
case Some(l: String) => {
cmdArgs.pageArgs += "libId" -> Some(l)
_libId = Some(l)
}
case _ =>
}
logger.info("Copy Base files")
// Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
......@@ -95,25 +127,32 @@ trait ReportBuilder extends ToolCommand {
logger.info("Parsing summary")
setSummary = new Summary(cmdArgs.summary)
total = countPages(indexPage)
total = ReportBuilder.countPages(indexPage)
logger.info(total + " pages to be generated")
logger.info("Generate pages")
val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
args = pageArgs ++ cmdArgs.pageArgs ++
args = pageArgs ++ cmdArgs.pageArgs.toMap ++
Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage))
logger.info(jobs + " Done")
}
/** This must be implemented, this will be the root page of the report */
def indexPage: ReportPage
/** This must be implemented, this will because the title of the report */
def reportName: String
def countPages(page: ReportPage): Int = {
page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
}
/**
* This method will render the page and the subpages recursivly
* @param summary The summary object
* @param page Page to render
* @param outputDir Root output dir of the report
* @param path Path from root to current page
* @param args Args to add to this sub page, are args from current page are passed automaticly
* @return Number of pages including all subpages that are rendered
*/
def generatePage(summary: Summary,
page: ReportPage,
outputDir: File,
......@@ -151,10 +190,17 @@ trait ReportBuilder extends ToolCommand {
object ReportBuilder {
/** Single template render engine, this will have a cache for all compile templates */
protected val engine = new TemplateEngine()
/** Cache of temp file for templates from the classpath / jar */
private var templateCache: Map[String, File] = Map()
/** This will give the total number of pages including all nested pages */
def countPages(page: ReportPage): Int = {
page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
}
/**
* This method will render a template that is located in the classpath / jar
* @param location location in the classpath / jar
......@@ -162,6 +208,12 @@ object ReportBuilder {
* @return Rendered result of template
*/
def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Logging.logger.info("Rendering: " + location)
if (location == "/nl/lumc/sasc/biopet/pipelines/carp/carpFront.ssp") {
println("hier dus")
}
val templateFile: File = templateCache.get(location) match {
case Some(template) => template
case _ => {
......
......@@ -2,6 +2,10 @@ package nl.lumc.sasc.biopet.core.report
/**
* Created by pjvan_thof on 3/27/15.
*
* @param subPages Subpages for this page
* @param sections Sections for this page
* @param args Arguments for this page, this arguments get passed to all section and subpages
*/
case class ReportPage(subPages: List[(String, ReportPage)],
sections: List[(String, ReportSection)],
......
......@@ -2,15 +2,18 @@ package nl.lumc.sasc.biopet.core.report
/**
* Created by pjvan_thof on 4/8/15.
*
* @param location Location inside the classpath / jar
* @param args arguments only for current section, this is not passed to other sub pages
*/
case class ReportSection(location: String,
args: Map[String, Any] = Map(),
intro: Option[String] = None) {
def render(args: Map[String, Any]): String = {
(intro match {
case Some(template) => ReportBuilder.renderTemplate(location, args ++ this.args)
case _ => ""
}) + ReportBuilder.renderTemplate(location, args ++ this.args)
args: Map[String, Any] = Map()) {
/**
* This method will render this section
* @param args Possible to give more arguments
* @return Rendered result for this section
*/
def render(args: Map[String, Any] = Map()): String = {
ReportBuilder.renderTemplate(location, args ++ this.args)
}
}
......@@ -10,6 +10,7 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils
class Summary(file: File) {
val map = ConfigUtils.fileToConfigMap(file)
/** List of all samples in the summary */
lazy val samples: Set[String] = {
ConfigUtils.getValueFromPath(map, List("samples")) match {
case Some(samples) => ConfigUtils.any2map(samples).keySet
......@@ -17,6 +18,7 @@ class Summary(file: File) {
}
}
/** List of all libraries for each sample */
lazy val libraries: Map[String, Set[String]] = {
(for (sample <- samples) yield sample -> {
ConfigUtils.getValueFromPath(map, List("samples", sample, "libraries")) match {
......@@ -26,27 +28,33 @@ class Summary(file: File) {
}).toMap
}
/** getValue from on given nested path */
def getValue(path: String*): Option[Any] = {
ConfigUtils.getValueFromPath(map, path.toList)
}
/** getValue from on given nested path with prefix "samples" -> [sampleId] */
def getSampleValue(sampleId: String, path: String*): Option[Any] = {
ConfigUtils.getValueFromPath(map, "samples" :: sampleId :: path.toList)
}
/** Get values for all samples on given path with prefix "samples" -> [sampleId] */
def getSampleValues(path: String*): Map[String, Option[Any]] = {
(for (sample <- samples) yield sample -> getSampleValue(sample, path: _*)).toMap
}
/** Executes given function for each sample */
def getSampleValues(function: (Summary, String) => Option[Any]): Map[String, Option[Any]] = {
(for (sample <- samples) yield sample -> function(this, sample)).toMap
}
/** Get value on nested path with prefix "samples" -> [sampleId] -> "libraries" -> [libId] */
def getLibraryValue(sampleId: String, libId: String, path: String*): Option[Any] = {
ConfigUtils.getValueFromPath(map, "samples" :: sampleId :: "libraries" :: libId :: path.toList)
}
def getLibraryValue(sampleId: Option[String], libId: Option[String], path: String*): Option[Any] = {
/** Get value on nested path with prefix depending is sampleId and/or libId is None or not */
def getValue(sampleId: Option[String], libId: Option[String], path: String*): Option[Any] = {
(sampleId, libId) match {