Commit 057f8873 authored by Sander Bollen's avatar Sander Bollen
Browse files

Merge branch 'fix-BIOPET-374' into 'develop'

Fix biopet 374



See merge request !476
parents ebebec1a b71f8de4
......@@ -83,10 +83,10 @@
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
#{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
val total = summary.getValue((prefixPath ::: List("biopet_flagstat", "All")):_*).getOrElse(0L).asInstanceOf[Long]
val mapped = summary.getValue((prefixPath ::: List("biopet_flagstat", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long]
val duplicates = summary.getValue((prefixPath ::: List("biopet_flagstat", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long]
val secondary = summary.getValue((prefixPath ::: List("biopet_flagstat", "NotPrimaryAlignment")):_*).getOrElse(0L).asInstanceOf[Long]
val total = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "All")):_*).getOrElse(0L).asInstanceOf[Long]
val mapped = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long]
val duplicates = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long]
val secondary = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "NotPrimaryAlignment")):_*).getOrElse(0L).asInstanceOf[Long]
}#
<td>${total}</td>
<td>${mapped}</td>
......
......@@ -12,9 +12,9 @@
#for (field <- fields)
<tr><th>${field}</th><td>
#if (libId.isDefined)
${summary.getLibraryValue(sampleId.get, libId.get, metricsTag, "stats", "biopet_flagstat", field)}
${summary.getLibraryValue(sampleId.get, libId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#else
${summary.getSampleValue(sampleId.get, metricsTag, "stats", "biopet_flagstat", field)}
${summary.getSampleValue(sampleId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#end
</td></tr>
#end
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File)
#import(org.apache.commons.io.FileUtils)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var sampleLevel: Boolean = false %>
<%@ var outputDir: File %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%>
#{
val samples = sampleId match {
case Some(sample) => {
List(sample.toString)
}
case _ => summary.samples.toList
}
}#
#if (showIntro)
<br/>
<div class="row">
<div class="col-md-1"></div>
<div class="col-md-6">
<p>
#if (sampleId.isDefined && libId.isDefined)
This plot shows the clipping distribution for library <b>${libId}</b> of sample <b>${sampleId}</b>.
#elseif(sampleId.isDefined)
This plot shows the clipping distribution for the libraries of sample <b>${sampleId}</b>.
#else
This plot shows the clipping distribution for each of the <b>${samples.size}</b> samples.
#end
</p>
</div>
</div>
#end
#if (showPlot)
#{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libId = libId) }#
<div class="panel-body">
<img src="clipping.png" class="img-responsive" />
</div>
<div class="panel-footer">
#if (showTable)
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#clippingTable">
<i class="glyphicon glyphicon-eye-close"></i> Hide table</button>
#else
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#clippingTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button>
#end
<a href="clipping.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i>TSV file</button></a>
</div>
#end
<div class="panel-body collapse #if (showTable)in#end" id="clippingTable">
<!-- Table -->
<table class="table">
<thead><tr>
<th data-sorted="true" data-sorted-direction="ascending">Sample</th>
#if (!sampleLevel) <th>Library</th> #end
#for (field <- fields)
<th>${field.replaceAll("_", " ")}</th>
#end
</tr></thead>
<tbody>
#for (sample <- samples.toList.sorted)
#{
val libs = (libId, sampleLevel) match {
case (_, true) => List("")
case (Some(libId), _) => List(libId.toString)
case _ => summary.libraries(sample).toList
}
}#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
#for (libId <- libs)
#if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
#{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
val fieldValues = for (field <- fields) yield {
summary.getValue((prefixPath ::: List("bamstats", "clipping", "general", field)):_*).getOrElse("N/A")
}
}#
#for (value <- fieldValues)
<td>${value}</td>
#end
</tr>
#end
#end
</tbody>
</table>
</div>
......@@ -51,12 +51,12 @@
<div class="panel-footer">
#if (showTable)
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#insertsizeTable">
<i class="glyphicon glyphicon-eye-close"></i> Hide table</button>
<i class="glyphicon glyphicon-eye-close"></i>Hide table</button>
#else
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#insertsizeTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button>
<i class="glyphicon glyphicon-eye-open"></i>Show table</button>
#end
<button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"> <a href="insertsize.tsv">tsv file</a></i></button>
<a href="insertsize.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i>TSV file</button></a>
</div>
#end
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File)
#import(org.apache.commons.io.FileUtils)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var sampleLevel: Boolean = false %>
<%@ var outputDir: File %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%>
#{
val samples = sampleId match {
case Some(sample) => {
List(sample.toString)
}
case _ => summary.samples.toList
}
}#
#if (showIntro)
<br/>
<div class="row">
<div class="col-md-1"></div>
<div class="col-md-6">
<p>
#if (sampleId.isDefined && libId.isDefined)
This plot shows the mapping quality distribution for all libraries combined in sample <b>${sampleId}</b>.
#elseif(sampleId.isDefined)
This plot shows the mapping quality distribution for the libraries of sample <b>${sampleId}</b>.
#else
This plot shows the mapping quality distribution for each of the <b>${samples.size}</b> samples.
#end
</p>
</div>
</div>
#end
#if (showPlot)
#{ BammetricsReport.mappingQualityPlot(outputDir, "mapping_quality", summary, !sampleLevel, sampleId = sampleId, libId = libId) }#
<div class="panel-body">
<img src="mapping_quality.png" class="img-responsive" />
</div>
<div class="panel-footer">
#if (showTable)
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#mapping_qualityTable">
<i class="glyphicon glyphicon-eye-close"></i> Hide table</button>
#else
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#mapping_qualityTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button>
#end
<a href="mapping_quality.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> TSV file</button></a>
</div>
#end
<div class="panel-body collapse #if (showTable)in#end" id="mapping_qualityTable">
<!-- Table -->
<table class="table">
<thead><tr>
<th data-sorted="true" data-sorted-direction="ascending">Sample</th>
#if (!sampleLevel) <th>Library</th> #end
#for (field <- fields)
<th>${field.replaceAll("_", " ")}</th>
#end
</tr></thead>
<tbody>
#for (sample <- samples.toList.sorted)
#{
val libs = (libId, sampleLevel) match {
case (_, true) => List("")
case (Some(libId), _) => List(libId.toString)
case _ => summary.libraries(sample).toList
}
}#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
#for (libId <- libs)
#if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
#{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
val fieldValues = for (field <- fields) yield {
summary.getValue((prefixPath ::: List("bamstats", "mapping_quality", "general", field)):_*).getOrElse("N/A")
}
}#
#for (value <- fieldValues)
<td>${value}</td>
#end
</tr>
#end
#end
</tbody>
</table>
</div>
......@@ -22,7 +22,7 @@ import nl.lumc.sasc.biopet.core.{ BiopetFifoPipe, PipelineCommand, Reference, Sa
import nl.lumc.sasc.biopet.extensions.bedtools.{ BedtoolsCoverage, BedtoolsIntersect, BedtoolsSort }
import nl.lumc.sasc.biopet.extensions.picard._
import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsFlagstat
import nl.lumc.sasc.biopet.extensions.tools.BiopetFlagstat
import nl.lumc.sasc.biopet.extensions.tools.{ BamStats, BiopetFlagstat }
import nl.lumc.sasc.biopet.pipelines.bammetrics.scripts.CoverageStats
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.intervals.BedCheck
......@@ -82,9 +82,11 @@ class BamMetrics(val root: Configurable) extends QScript
def biopetScript() {
add(SamtoolsFlagstat(this, inputBam, outputDir))
val biopetFlagstat = BiopetFlagstat(this, inputBam, outputDir)
add(biopetFlagstat)
addSummarizable(biopetFlagstat, "biopet_flagstat")
val bamStats = new BamStats(this)
bamStats.bamFile = inputBam
bamStats.outputDir = new File(outputDir, "bamstats")
add(bamStats)
addSummarizable(bamStats, "bamstats")
val multiMetrics = new CollectMultipleMetrics(this)
multiMetrics.input = inputBam
......
......@@ -82,7 +82,9 @@ object BammetricsReport extends ReportBuilder {
targets.map(t => t -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp", Map("target" -> Some(t)))),
Map())),
List(
"Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp")) ++
"Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"),
"Mapping Quality" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/mappingQuality.ssp", Map("showPlot" -> true)),
"Clipping" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/clipping.ssp", Map("showPlot" -> true))) ++
(if (insertsizeMetrics) List("Insert Size" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/insertSize.ssp", Map("showPlot" -> true))
)
else Nil) ++ (if (wgsExecuted) List("Whole genome coverage" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/wgsHistogram.ssp",
......@@ -176,76 +178,80 @@ object BammetricsReport extends ReportBuilder {
libId: Option[String] = None): Unit = {
val tsvFile = new File(outputDir, prefix + ".tsv")
val pngFile = new File(outputDir, prefix + ".png")
val tsvWriter = new PrintWriter(tsvFile)
if (libraryLevel) {
tsvWriter.println((for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) yield s"$sample-$lib")
.mkString("library\t", "\t", ""))
} else {
sampleId match {
case Some(sample) => tsvWriter.println("\t" + sample)
case _ => tsvWriter.println(summary.samples.mkString("Sample\t", "\t", ""))
}
}
var map: Map[Int, Map[String, Int]] = Map()
def fill(sample: String, lib: Option[String]): Unit = {
val insertSize = new SummaryValue(List("bammetrics", "stats", "CollectInsertSizeMetrics", "histogram", "insert_size"),
summary, Some(sample), lib).value.getOrElse(List())
val counts = new SummaryValue(List("bammetrics", "stats", "CollectInsertSizeMetrics", "histogram", "All_Reads.fr_count"),
summary, Some(sample), lib).value.getOrElse(List())
(insertSize, counts) match {
case (l: List[_], l2: List[_]) =>
l.zip(l2).foreach(i => {
val insertSize = i._1.toString.toInt
val count = i._2.toString.toInt
val old = map.getOrElse(insertSize, Map())
if (libraryLevel) map += insertSize -> (old + ((s"$sample-" + lib.get) -> count))
else map += insertSize -> (old + (sample -> count))
})
case _ => throw new IllegalStateException("Must be a list")
def paths(name: String) = Map(
"insert_size" -> List("bammetrics", "stats", "CollectInsertSizeMetrics", "histogram", "insert_size"),
name -> List("bammetrics", "stats", "CollectInsertSizeMetrics", "histogram", "All_Reads.fr_count")
)
val tables = getSampleLibraries(summary, sampleId, libId, libraryLevel)
.map {
case (sample, lib) =>
getTableFromSummary(summary, paths(lib.map(l => s"$sample-$l").getOrElse(sample)), Some(sample), lib)
}
}
writeTableToTsv(tsvFile, mergeTables(tables.toArray, "insert_size"), "insert_size")
if (libraryLevel) {
for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) fill(sample, Some(lib))
} else if (sampleId.isDefined) fill(sampleId.get, None)
else summary.samples.foreach(fill(_, None))
for ((insertSize, counts) <- map) {
tsvWriter.print(insertSize)
if (libraryLevel) {
for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) tsvWriter.print("\t" + counts.getOrElse(s"$sample-$lib", "0"))
} else {
for (sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample) {
tsvWriter.print("\t" + counts.getOrElse(sample, "0"))
}
LinePlot(tsvFile, pngFile,
xlabel = Some("Insert size"),
ylabel = Some("Reads"),
title = Some("Insert size"),
removeZero = true).runLocal()
}
def mappingQualityPlot(outputDir: File,
prefix: String,
summary: Summary,
libraryLevel: Boolean = false,
sampleId: Option[String] = None,
libId: Option[String] = None): Unit = {
val tsvFile = new File(outputDir, prefix + ".tsv")
val pngFile = new File(outputDir, prefix + ".png")
def paths(name: String) = Map(
"mapping_quality" -> List("bammetrics", "stats", "bamstats", "mapping_quality", "histogram", "value"),
name -> List("bammetrics", "stats", "bamstats", "mapping_quality", "histogram", "count")
)
val tables = getSampleLibraries(summary, sampleId, libId, libraryLevel)
.map {
case (sample, lib) =>
getTableFromSummary(summary, paths(lib.map(l => s"$sample-$l").getOrElse(sample)), Some(sample), lib)
}
tsvWriter.println()
}
writeTableToTsv(tsvFile, mergeTables(tables.toArray, "mapping_quality"), "mapping_quality")
tsvWriter.close()
LinePlot(tsvFile, pngFile,
xlabel = Some("Mapping Quality"),
ylabel = Some("Reads"),
title = Some("Mapping Quality"),
removeZero = true).runLocal()
}
val plot = new LinePlot(null)
plot.input = tsvFile
plot.output = pngFile
plot.ylabel = Some("Reads")
plot.xlabel = Some("Insert size")
plot.width = Some(1200)
plot.removeZero = true
plot.title = Some("Insert size")
plot.runLocal()
def clippingPlot(outputDir: File,
prefix: String,
summary: Summary,
libraryLevel: Boolean = false,
sampleId: Option[String] = None,
libId: Option[String] = None): Unit = {
val tsvFile = new File(outputDir, prefix + ".tsv")
val pngFile = new File(outputDir, prefix + ".png")
def paths(name: String) = Map(
"clipping" -> List("bammetrics", "stats", "bamstats", "clipping", "histogram", "value"),
name -> List("bammetrics", "stats", "bamstats", "clipping", "histogram", "count")
)
val tables = getSampleLibraries(summary, sampleId, libId, libraryLevel)
.map {
case (sample, lib) =>
getTableFromSummary(summary, paths(lib.map(l => s"$sample-$l").getOrElse(sample)), Some(sample), lib)
}
writeTableToTsv(tsvFile, mergeTables(tables.toArray, "clipping"), "clipping")
LinePlot(tsvFile, pngFile,
xlabel = Some("Clipping"),
ylabel = Some("Reads"),
title = Some("Clipping"),
removeZero = true).runLocal()
}
/**
......@@ -265,78 +271,24 @@ object BammetricsReport extends ReportBuilder {
libId: Option[String] = None): Unit = {
val tsvFile = new File(outputDir, prefix + ".tsv")
val pngFile = new File(outputDir, prefix + ".png")
val tsvWriter = new PrintWriter(tsvFile)
if (libraryLevel) {
tsvWriter.println((for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) yield s"$sample-$lib")
.mkString("library\t", "\t", ""))
} else {
sampleId match {
case Some(sample) => tsvWriter.println("\t" + sample)
case _ => tsvWriter.println(summary.samples.mkString("Sample\t", "\t", ""))
}
}
var map: Map[Int, Map[String, Long]] = Map()
def fill(sample: String, lib: Option[String]): Unit = {
val insertSize = new SummaryValue(List("bammetrics", "stats", "wgs", "histogram", "coverage"),
summary, Some(sample), lib).value.getOrElse(List())
val counts = new SummaryValue(List("bammetrics", "stats", "wgs", "histogram", "count"),
summary, Some(sample), lib).value.getOrElse(List())
(insertSize, counts) match {
case (l: List[_], l2: List[_]) =>
l.zip(l2).foreach(i => {
val insertSize = i._1.toString.toInt
val count = i._2.toString.toLong
val old = map.getOrElse(insertSize, Map())
if (libraryLevel) map += insertSize -> (old + ((s"$sample-" + lib.get) -> count))
else map += insertSize -> (old + (sample -> count))
})
case _ => throw new IllegalStateException("Must be a list")
}
}
def paths(name: String) = Map(
"coverage" -> List("bammetrics", "stats", "wgs", "histogram", "coverage"),
name -> List("bammetrics", "stats", "wgs", "histogram", "count")
)
if (libraryLevel) {
for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) fill(sample, Some(lib))
} else if (sampleId.isDefined) fill(sampleId.get, None)
else summary.samples.foreach(fill(_, None))
for ((insertSize, counts) <- map) {
tsvWriter.print(insertSize)
if (libraryLevel) {
for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) {
tsvWriter.print("\t" + counts.getOrElse(s"$sample-$lib", "0"))
}
} else {
for (sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample) {
tsvWriter.print("\t" + counts.getOrElse(sample, "0"))
}
val tables = getSampleLibraries(summary, sampleId, libId, libraryLevel)
.map {
case (sample, lib) =>
getTableFromSummary(summary, paths(lib.map(l => s"$sample-$l").getOrElse(sample)), Some(sample), lib)
}
tsvWriter.println()
}
tsvWriter.close()
writeTableToTsv(tsvFile, mergeTables(tables.toArray, "coverage"), "coverage")
val plot = new LinePlot(null)
plot.input = tsvFile
plot.output = pngFile
plot.ylabel = Some("Bases")
plot.xlabel = Some("Coverage")
plot.width = Some(1200)
plot.removeZero = true
plot.title = Some("Whole genome coverage")
plot.runLocal()
LinePlot(tsvFile, pngFile,
xlabel = Some("Coverage"),
ylabel = Some("Bases"),
title = Some("Whole genome coverage"),
removeZero = true).runLocal()
}
/**
......@@ -356,77 +308,67 @@ object BammetricsReport extends ReportBuilder {
libId: Option[String] = None): Unit = {
val tsvFile = new File(outputDir, prefix + ".tsv")
val pngFile = new File(outputDir, prefix + ".png")
val tsvWriter = new PrintWriter(tsvFile)
if (libraryLevel) {
tsvWriter.println((for (
sample <- summary.samples if sampleId.isEmpty || sampleId.get == sample;
lib <- summary.libraries(sample) if libId.isEmpty || libId.get == lib
) yield s"$sample-$lib")
.mkString("library\t", "\t", ""))
} else {
sampleId match {
case Some(sample) => tsvWriter.println("\t" + sample)
case _ => tsvWriter.println(summary.samples.mkString("Sample\t", "\t", ""))
}
}
var map: Map[Int, Map[String, Double]] = Map()
def fill(sample: String, lib: Option[String]): Unit = {
val insertSize = new SummaryValue(List("bammetrics", "stats", "rna", "histogram", "normalized_position"),
summary, Some(sample), lib).value.getOrElse(List())
val counts = new SummaryValue(List("bammetrics", "stats", "rna", "histogram", "All_Reads.normalized_coverage"),
summary, Some(sample), lib).value.getOrElse(List())
(insertSize, counts) match {
case (l: List[_], l2: List[_]) =>
l.zip(l2).foreach(i => {
val insertSize = i._1.toString.toInt
val count = i._2.toString.toDouble
val old = map.getOrElse(insertSize, Map())
if (libraryLevel) map += insertSize -> (old + ((s"$sample-" + lib.get) -> count))
else map += insertSize -> (old + (sample -> count))
})
case _ => throw new IllegalStateException("Must be a list")
}
}