...
 
Commits (761)
#Files to be included
project.git = true #All scala files tracked by git.
project.excludeFilters = [
"gatk/*"
]
style = default
maxColumn = 100
...@@ -21,10 +21,6 @@ node('local') { ...@@ -21,10 +21,6 @@ node('local') {
} }
} }
stage('Report Tests') {
junit '*/target/surefire-reports/*.xml'
}
stage('Check git on changes') { stage('Check git on changes') {
sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi' sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
} }
...@@ -33,7 +29,7 @@ node('local') { ...@@ -33,7 +29,7 @@ node('local') {
sh 'mkdocs build --clean --strict' sh 'mkdocs build --clean --strict'
} }
if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) { if (currentBuild.result == null || "SUCCESS" == currentBuild.result) {
currentBuild.result = "SUCCESS" currentBuild.result = "SUCCESS"
slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc') slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else { } else {
...@@ -41,7 +37,7 @@ node('local') { ...@@ -41,7 +37,7 @@ node('local') {
} }
} catch (e) { } catch (e) {
if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) { if (currentBuild.result == null || "FAILED" == currentBuild.result) {
currentBuild.result = "FAILED" currentBuild.result = "FAILED"
} }
slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc') slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
<parent> <parent>
<groupId>nl.lumc.sasc</groupId> <groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId> <artifactId>Biopet</artifactId>
<version>0.9.0</version> <version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
......
...@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.pipelines.bamtobigwig ...@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.pipelines.bamtobigwig
import java.io.File import java.io.File
import nl.lumc.sasc.biopet.utils.config.Configurable import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.{ BiopetQScript, PipelineCommand } import nl.lumc.sasc.biopet.core.{BiopetQScript, PipelineCommand}
import nl.lumc.sasc.biopet.extensions.WigToBigWig import nl.lumc.sasc.biopet.extensions.WigToBigWig
import nl.lumc.sasc.biopet.extensions.igvtools.IGVToolsCount import nl.lumc.sasc.biopet.extensions.igvtools.IGVToolsCount
import org.broadinstitute.gatk.queue.QScript import org.broadinstitute.gatk.queue.QScript
...@@ -31,7 +31,7 @@ class Bam2Wig(val parent: Configurable) extends QScript with BiopetQScript { ...@@ -31,7 +31,7 @@ class Bam2Wig(val parent: Configurable) extends QScript with BiopetQScript {
def this() = this(null) def this() = this(null)
@Input(doc = "Input bam file", required = true) @Input(doc = "Input bam file", required = true)
var bamFile: File = null var bamFile: File = _
def init(): Unit = { def init(): Unit = {
inputFiles :+= new InputFile(bamFile) inputFiles :+= new InputFile(bamFile)
......
...@@ -14,12 +14,12 @@ ...@@ -14,12 +14,12 @@
*/ */
package nl.lumc.sasc.biopet.pipelines.bamtobigwig package nl.lumc.sasc.biopet.pipelines.bamtobigwig
import java.io.{ File, PrintWriter } import java.io.{File, PrintWriter}
import htsjdk.samtools.SamReaderFactory import htsjdk.samtools.SamReaderFactory
import nl.lumc.sasc.biopet.utils.config.Configurable import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.queue.function.InProcessFunction import org.broadinstitute.gatk.queue.function.InProcessFunction
import org.broadinstitute.gatk.utils.commandline.{ Input, Output } import org.broadinstitute.gatk.utils.commandline.{Input, Output}
import scala.collection.JavaConversions._ import scala.collection.JavaConversions._
......
...@@ -32,13 +32,14 @@ class BamToChromSizesTest extends TestNGSuite with Matchers { ...@@ -32,13 +32,14 @@ class BamToChromSizesTest extends TestNGSuite with Matchers {
} }
@Test @Test
def testChromSizes: Unit = { def testChromSizes(): Unit = {
val bamFile = new File(resourcePath("/empty.bam")) val bamFile = new File(resourcePath("/empty.bam"))
val bamToChromSizes = new BamToChromSizes(null) val bamToChromSizes = new BamToChromSizes(null)
bamToChromSizes.bamFile = bamFile bamToChromSizes.bamFile = bamFile
bamToChromSizes.chromSizesFile = File.createTempFile("chrom.", ".sizes") bamToChromSizes.chromSizesFile = File.createTempFile("chrom.", ".sizes")
bamToChromSizes.chromSizesFile.deleteOnExit() bamToChromSizes.chromSizesFile.deleteOnExit()
bamToChromSizes.run() bamToChromSizes.run()
Source.fromFile(bamToChromSizes.chromSizesFile).getLines().toList shouldBe List("chrQ\t10000", "chrR\t10000") Source.fromFile(bamToChromSizes.chromSizesFile).getLines().toList shouldBe List("chrQ\t10000",
"chrR\t10000")
} }
} }
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
<parent> <parent>
<groupId>nl.lumc.sasc</groupId> <groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId> <artifactId>Biopet</artifactId>
<version>0.9.0</version> <version>0.10.0-SNAPSHOT</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
...@@ -39,7 +39,7 @@ ...@@ -39,7 +39,7 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>nl.lumc.sasc</groupId> <groupId>nl.lumc.sasc</groupId>
<artifactId>BiopetToolsExtensions</artifactId> <artifactId>BiopetExtensions</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
</dependencies> </dependencies>
......
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
...@@ -19,6 +20,8 @@ ...@@ -19,6 +20,8 @@
<%@ var runId: Int %> <%@ var runId: Int %>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var alignmentSummaryPlotLines: Option[Seq[String]] %>
<%@ var alignmentSummaryResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -45,7 +48,7 @@ ...@@ -45,7 +48,7 @@
</div> </div>
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.alignmentSummaryPlot(outputDir, "alignmentSummary", summary, !sampleLevel, sampleId = sampleId) }# #{ BammetricsReport.alignmentSummaryPlot(outputDir, "alignmentSummary", alignmentSummaryPlotLines.getOrElse(Seq("")), !sampleLevel) }#
<div class="panel-body"> <div class="panel-body">
<img src="alignmentSummary.png" class="img-responsive" /> <img src="alignmentSummary.png" class="img-responsive" />
...@@ -58,7 +61,7 @@ ...@@ -58,7 +61,7 @@
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#alignmentSummaryTable"> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#alignmentSummaryTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button> <i class="glyphicon glyphicon-eye-open"></i> Show table</button>
#end #end
<a href="alignmentSummary.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> TSV file</button></a> <a href="alignmentSummary.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
<div class="panel-body collapse #if (showTable)in#end" id="alignmentSummaryTable"> <div class="panel-body collapse #if (showTable)in#end" id="alignmentSummaryTable">
...@@ -95,11 +98,10 @@ ...@@ -95,11 +98,10 @@
"Duplicates" -> List("flagstats", "Duplicates"), "Duplicates" -> List("flagstats", "Duplicates"),
"NotPrimaryAlignment" -> List("flagstats", "NotPrimaryAlignment") "NotPrimaryAlignment" -> List("flagstats", "NotPrimaryAlignment")
) )
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths) val total = alignmentSummaryResults((sample.id,libId.get))("All").getOrElse(0L).asInstanceOf[Long]
val total = results("All").getOrElse(0L).asInstanceOf[Long] val mapped = alignmentSummaryResults((sample.id, libId.get))("Mapped").getOrElse(0L).asInstanceOf[Long]
val mapped = results("Mapped").getOrElse(0L).asInstanceOf[Long] val duplicates = alignmentSummaryResults((sample.id,libId.get))("Duplicates").getOrElse(0L).asInstanceOf[Long]
val duplicates = results("Duplicates").getOrElse(0L).asInstanceOf[Long] val secondary = alignmentSummaryResults((sample.id,libId.get))("NotPrimaryAlignment").getOrElse(0L).asInstanceOf[Long]
val secondary = results("NotPrimaryAlignment").getOrElse(0L).asInstanceOf[Long]
}# }#
<td>${total}</td> <td>${total}</td>
<td>${mapped}</td> <td>${mapped}</td>
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
<tr><th>Version</th><td>${run.version}</td></tr> <tr><th>Version</th><td>${run.version}</td></tr>
<tr><th>Last commit hash</th><td>${run.commitHash}</td></tr> <tr><th>Last commit hash</th><td>${run.commitHash}</td></tr>
<tr><th>Output directory</th><td>${run.outputDir}</td></tr> <tr><th>Output directory</th><td>${run.outputDir}</td></tr>
<tr><th>Sample</th><td>${allSamples.filter(_.id == sampleId.get).headOption.map(_.name)}</td></tr> <tr><th>Sample</th><td>${allSamples.find(_.id == sampleId.get).map(_.name)}</td></tr>
<tr><th>Library</th><td>${allLibraries.filter(_.id == libId.get).headOption.map(_.name)}</td></tr> <tr><th>Library</th><td>${allLibraries.find(_.id == libId.get).map(_.name)}</td></tr>
</tbody> </tbody>
</table> </table>
<br/> <br/>
......
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
...@@ -18,6 +19,8 @@ ...@@ -18,6 +19,8 @@
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%> <%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var clippingPlotTables: Option[Array[Map[String, Array[Any]]]] %>
<%@ var clippingTableResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -43,8 +46,7 @@ ...@@ -43,8 +46,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }# #{ BammetricsReport.clippingPlot(outputDir, "clipping", clippingPlotTables.getOrElse(Array(Map("x" -> Array(""))))) }#
<div class="panel-body"> <div class="panel-body">
<img src="clipping.png" class="img-responsive" /> <img src="clipping.png" class="img-responsive" />
</div> </div>
...@@ -56,7 +58,7 @@ ...@@ -56,7 +58,7 @@
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#clippingTable"> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#clippingTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button> <i class="glyphicon glyphicon-eye-open"></i> Show table</button>
#end #end
<a href="clipping.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i>TSV file</button></a> <a href="clipping.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
...@@ -86,8 +88,7 @@ ...@@ -86,8 +88,7 @@
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val statsPaths = fields.map(x => x -> List("clipping", "general", x)).toMap val results: Map[String, Option[Any]] = clippingTableResults((sample.id,libId.get))
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}# }#
#for (field <- fields) #for (field <- fields)
<td>${results(field)}</td> <td>${results(field)}</td>
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#import(scala.concurrent.Await) #import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration) #import(scala.concurrent.duration.Duration)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport) #import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema)
#import(java.io.File) #import(java.io.File)
<%@ var summary: SummaryDb %> <%@ var summary: SummaryDb %>
<%@ var sampleId: Option[Int] %> <%@ var sampleId: Option[Int] %>
...@@ -16,16 +17,14 @@ ...@@ -16,16 +17,14 @@
<%@ var target: Option[String] %> <%@ var target: Option[String] %>
<%@ var runId: Int %> <%@ var runId: Int %>
<%@ var fields: List[String] = List("mean", "median", "max", "horizontal", "frac_min_10x", "frac_min_20x", "frac_min_30x", "frac_min_40x", "frac_min_50x") %> <%@ var fields: List[String] = List("mean", "median", "max", "horizontal", "frac_min_10x", "frac_min_20x", "frac_min_30x", "frac_min_40x", "frac_min_50x") %>
<%@ var plotFile: Option[Schema.File] %>
<%@ var values: Map[String, Option[Any]] %>
#{ #{
val moduleName = target.get + "_cov_stats" val moduleName = target.get + "_cov_stats"
val statsPaths = fields.map(x => x -> List("coverage", "_all", x)).toMap
val plotFile = Await.result(summary.getFile(runId, PipelineName(metricsTag), ModuleName(moduleName), sampleId.map(SampleId).get, libId.map(LibraryId).getOrElse(NoLibrary), "plot"), Duration.Inf)
val originalPlot = new File(plotFile.get.path) val originalPlot = new File(plotFile.get.path)
val plot = new File(outputDir, target.get + "_cov_stats.png") val plot = new File(outputDir, target.get + "_cov_stats.png")
val values = summary.getStatKeys(runId, PipelineName(metricsTag), ModuleName(moduleName), sampleId.map(SampleId).get, libId.map(LibraryId).getOrElse(NoLibrary), statsPaths)
if (originalPlot.exists()) IoUtils.copyFile(originalPlot, plot) if (originalPlot.exists()) IoUtils.copyFile(originalPlot, plot)
}# }#
<img src="${plot.getName}"> <img src="${plot.getName}">
......
...@@ -9,7 +9,6 @@ ...@@ -9,7 +9,6 @@
<%@ var sampleId: Option[Int] = None %> <%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[Int] = None %> <%@ var libId: Option[Int] = None %>
<%@ var rootPath: String %> <%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var sampleLevel: Boolean = false %> <%@ var sampleLevel: Boolean = false %>
<%@ var outputDir: File %> <%@ var outputDir: File %>
<%@ var fields: List[String] = List("mean_insert_size", "standard_deviation", "median_insert_size")%> <%@ var fields: List[String] = List("mean_insert_size", "standard_deviation", "median_insert_size")%>
...@@ -19,6 +18,8 @@ ...@@ -19,6 +18,8 @@
<%@ var runId: Int %> <%@ var runId: Int %>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var insertSizePlotTables: Option[Array[Map[String, Array[Any]]]] %>
<%@ var insertSizeTableResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -45,7 +46,7 @@ ...@@ -45,7 +46,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.insertSizePlot(outputDir, "insertsize", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }# #{ BammetricsReport.insertSizePlot(outputDir, "insertsize", insertSizePlotTables.getOrElse(Array(Map("x" -> Array(""))))) }#
<div class="panel-body"> <div class="panel-body">
<img src="insertsize.png" class="img-responsive" /> <img src="insertsize.png" class="img-responsive" />
...@@ -58,7 +59,7 @@ ...@@ -58,7 +59,7 @@
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#insertsizeTable"> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#insertsizeTable">
<i class="glyphicon glyphicon-eye-open"></i>Show table</button> <i class="glyphicon glyphicon-eye-open"></i>Show table</button>
#end #end
<a href="insertsize.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i>TSV file</button></a> <a href="insertsize.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
...@@ -88,8 +89,7 @@ ...@@ -88,8 +89,7 @@
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val statsPaths = fields.map(x => x -> List("metrics", x.toUpperCase)).toMap val results: Map[String, Option[Any]] = insertSizeTableResults((sample.id, libId.get))
val results = summary.getStatKeys(runId, "bammetrics", "CollectInsertSizeMetrics", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}# }#
#for (field <- fields) #for (field <- fields)
<td>${results(field)}</td> <td>${results(field)}</td>
......
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._) #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage) #import(nl.lumc.sasc.biopet.core.report.ReportPage)
...@@ -18,6 +19,8 @@ ...@@ -18,6 +19,8 @@
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%> <%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var mappingQualityPlotTables: Option[Array[Map[String, Array[Any]]]] %>
<%@ var mappingQualityTableResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -43,7 +46,7 @@ ...@@ -43,7 +46,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.mappingQualityPlot(outputDir, "mapping_quality", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }# #{ BammetricsReport.mappingQualityPlot(outputDir, "mapping_quality", mappingQualityPlotTables.getOrElse(Array(Map("x" -> Array("")))))}#
<div class="panel-body"> <div class="panel-body">
<img src="mapping_quality.png" class="img-responsive" /> <img src="mapping_quality.png" class="img-responsive" />
...@@ -56,7 +59,7 @@ ...@@ -56,7 +59,7 @@
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#mapping_qualityTable"> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#mapping_qualityTable">
<i class="glyphicon glyphicon-eye-open"></i> Show table</button> <i class="glyphicon glyphicon-eye-open"></i> Show table</button>
#end #end
<a href="mapping_quality.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> TSV file</button></a> <a href="mapping_quality.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
...@@ -86,8 +89,7 @@ ...@@ -86,8 +89,7 @@
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val statsPaths = fields.map(x => x -> List("mapping_quality", "general", x)).toMap val results: Map[String, Option[Any]] = mappingQualityTableResults((sample.id,libId.get))
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}# }#
#for (field <- fields) #for (field <- fields)
<td>${results(field)}</td> <td>${results(field)}</td>
......
...@@ -20,6 +20,8 @@ ...@@ -20,6 +20,8 @@
<%@ var runId: Int %> <%@ var runId: Int %>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var rnaHistogramPlotTables: Option[Array[Map[String, Array[Any]]]] %>
<%@ var rnaHistogramTableResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -39,7 +41,7 @@ ...@@ -39,7 +41,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.rnaHistogramPlot(outputDir, "rna", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }# #{ BammetricsReport.rnaHistogramPlot(outputDir, "rna", rnaHistogramPlotTables.getOrElse(Array(Map("x" -> Array(""))))) }#
<div class="panel-body"> <div class="panel-body">
<img src="rna.png" class="img-responsive" /> <img src="rna.png" class="img-responsive" />
...@@ -50,7 +52,7 @@ ...@@ -50,7 +52,7 @@
#else #else
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#rnaTable">Show table</button> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#rnaTable">Show table</button>
#end #end
<a href="rna.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i>TSV file</button></a> <a href="rna.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
...@@ -79,8 +81,7 @@ ...@@ -79,8 +81,7 @@
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val statsPaths = fields.map(x => x -> List("metrics", x.toUpperCase)).toMap val results = rnaHistogramTableResults((sample.id,libId.get))
val results = summary.getStatKeys(runId, "bammetrics", "rna", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}# }#
#for (field <- fields) #for (field <- fields)
<td>${results(field)}</td> <td>${results(field)}</td>
......
...@@ -19,6 +19,8 @@ ...@@ -19,6 +19,8 @@
<%@ var fields: List[String] = List("mean_coverage", "pct_5x", "pct_10x", "pct_15x", "pct_20x", "pct_25x", "pct_30x", "pct_40x", "pct_50x", "pct_60x", "pct_70x", "pct_80x", "pct_90x", "pct_100x")%> <%@ var fields: List[String] = List("mean_coverage", "pct_5x", "pct_10x", "pct_15x", "pct_20x", "pct_25x", "pct_30x", "pct_40x", "pct_50x", "pct_60x", "pct_70x", "pct_80x", "pct_90x", "pct_100x")%>
<%@ var allSamples: Seq[Sample] %> <%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %> <%@ var allLibraries: Seq[Library] %>
<%@ var wgsHistogramPlotTables: Option[Array[Map[String, Array[Any]]]] %>
<%@ var wgsHistogramTableResults: Map[(Int, Int), Map[String, Option[Any]]] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(id) => allSamples.filter(_.id == id).toList case Some(id) => allSamples.filter(_.id == id).toList
...@@ -38,7 +40,7 @@ ...@@ -38,7 +40,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.wgsHistogramPlot(outputDir, "wgs", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }# #{ BammetricsReport.wgsHistogramPlot(outputDir, "rna", wgsHistogramPlotTables.getOrElse(Array(Map("x" -> Array(""))))) }#
<div class="panel-body"> <div class="panel-body">
<img src="wgs.png" class="img-responsive" /> <img src="wgs.png" class="img-responsive" />
...@@ -49,7 +51,7 @@ ...@@ -49,7 +51,7 @@
#else #else
<button type="button" class="btn btn-info" data-toggle="collapse" data-target="#wgsTable">Show table</button> <button type="button" class="btn btn-info" data-toggle="collapse" data-target="#wgsTable">Show table</button>
#end #end
<a href="wgs.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> TSV file</button></a> <a href="wgs.tsv"><button type="button" class="btn btn-info"><i class="glyphicon glyphicon-cloud-download"></i> Data points</button></a>
</div> </div>
#end #end
...@@ -78,8 +80,7 @@ ...@@ -78,8 +80,7 @@
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val statsPaths = fields.map(x => x -> List("metrics", x.toUpperCase)).toMap val results = wgsHistogramTableResults((sample.id,libId.get))
val results = summary.getStatKeys(runId, "bammetrics", "wgs", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}# }#
#for (field <- fields) #for (field <- fields)
<td>${results(field)}</td> <td>${results(field)}</td>
......
...@@ -16,19 +16,20 @@ package nl.lumc.sasc.biopet.pipelines.bammetrics ...@@ -16,19 +16,20 @@ package nl.lumc.sasc.biopet.pipelines.bammetrics
import java.io.File import java.io.File
import nl.lumc.sasc.biopet.core.annotations.{ AnnotationRefFlat, RibosomalRefFlat } import nl.lumc.sasc.biopet.core.annotations.{AnnotationRefFlat, RibosomalRefFlat}
import nl.lumc.sasc.biopet.core.summary.SummaryQScript import nl.lumc.sasc.biopet.core.summary.SummaryQScript
import nl.lumc.sasc.biopet.core.{ BiopetFifoPipe, PipelineCommand, Reference, SampleLibraryTag } import nl.lumc.sasc.biopet.core.{BiopetFifoPipe, PipelineCommand, Reference, SampleLibraryTag}
import nl.lumc.sasc.biopet.extensions.bedtools.{ BedtoolsCoverage, BedtoolsIntersect, BedtoolsSort } import nl.lumc.sasc.biopet.extensions.bedtools.{BedtoolsCoverage, BedtoolsIntersect, BedtoolsSort}
import nl.lumc.sasc.biopet.extensions.picard._ import nl.lumc.sasc.biopet.extensions.picard._
import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsFlagstat import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsFlagstat
import nl.lumc.sasc.biopet.extensions.tools.{ BamStats, BiopetFlagstat } import nl.lumc.sasc.biopet.extensions.tools.{BamStats, BiopetFlagstat}
import nl.lumc.sasc.biopet.pipelines.bammetrics.scripts.CoverageStats import nl.lumc.sasc.biopet.pipelines.bammetrics.scripts.CoverageStats
import nl.lumc.sasc.biopet.utils.config.Configurable import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.intervals.BedCheck import nl.lumc.sasc.biopet.utils.intervals.BedCheck
import org.broadinstitute.gatk.queue.QScript import org.broadinstitute.gatk.queue.QScript
class BamMetrics(val parent: Configurable) extends QScript class BamMetrics(val parent: Configurable)
extends QScript
with SummaryQScript with SummaryQScript
with SampleLibraryTag with SampleLibraryTag
with Reference with Reference
...@@ -41,25 +42,29 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -41,25 +42,29 @@ class BamMetrics(val parent: Configurable) extends QScript
@Input(doc = "Bam File", shortName = "BAM", required = true) @Input(doc = "Bam File", shortName = "BAM", required = true)
var inputBam: File = _ var inputBam: File = _
@Argument(required = false)
var paired: Boolean = true
override def defaults = Map("bedtoolscoverage" -> Map("sorted" -> true)) override def defaults = Map("bedtoolscoverage" -> Map("sorted" -> true))
/** returns files to store in summary */ /** returns files to store in summary */
def summaryFiles = Map("reference" -> referenceFasta(), def summaryFiles: Map[String, File] =
"input_bam" -> inputBam) ++ Map("reference" -> referenceFasta(), "input_bam" -> inputBam) ++
ampliconBedFile.map("amplicon" -> _).toMap ++ ampliconBedFile.map("amplicon" -> _).toMap ++
ampliconBedFile.map(x => "roi_" + x.getName.stripSuffix(".bed") -> x).toMap ampliconBedFile.map(x => "roi_" + x.getName.stripSuffix(".bed") -> x).toMap
/** return settings */ /** return settings */
def summarySettings = Map("amplicon_name" -> ampliconBedFile.collect { case x => x.getName.stripSuffix(".bed") }, def summarySettings =
Map("amplicon_name" -> ampliconBedFile.collect { case x => x.getName.stripSuffix(".bed") },
"roi_name" -> roiBedFiles.map(_.getName.stripSuffix(".bed"))) "roi_name" -> roiBedFiles.map(_.getName.stripSuffix(".bed")))
override def reportClass = { override def reportClass: Some[BammetricsReport] = {
val bammetricsReport = new BammetricsReport(this) val bammetricsReport = new BammetricsReport(this)
bammetricsReport.outputDir = new File(outputDir, "report") bammetricsReport.outputDir = new File(outputDir, "report")
bammetricsReport.summaryDbFile = summaryDbFile bammetricsReport.summaryDbFile = summaryDbFile
bammetricsReport.args = if (libId.isDefined) Map( bammetricsReport.args =
"sampleId" -> sampleId.getOrElse("."), if (libId.isDefined)
"libId" -> libId.getOrElse(".")) Map("sampleId" -> sampleId.getOrElse("."), "libId" -> libId.getOrElse("."))
else Map("sampleId" -> sampleId.getOrElse(".")) else Map("sampleId" -> sampleId.getOrElse("."))
Some(bammetricsReport) Some(bammetricsReport)
} }
...@@ -67,7 +72,8 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -67,7 +72,8 @@ class BamMetrics(val parent: Configurable) extends QScript
/** executed before script */ /** executed before script */
def init(): Unit = { def init(): Unit = {
inputFiles :+= new InputFile(inputBam) inputFiles :+= new InputFile(inputBam)
ampliconBedFile.foreach(BedCheck.checkBedFileToReference(_, referenceFasta(), biopetError = true)) ampliconBedFile.foreach(
BedCheck.checkBedFileToReference(_, referenceFasta(), biopetError = true))
roiBedFiles.foreach(BedCheck.checkBedFileToReference(_, referenceFasta(), biopetError = true)) roiBedFiles.foreach(BedCheck.checkBedFileToReference(_, referenceFasta(), biopetError = true))
} }
...@@ -84,6 +90,9 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -84,6 +90,9 @@ class BamMetrics(val parent: Configurable) extends QScript
val multiMetrics = new CollectMultipleMetrics(this) val multiMetrics = new CollectMultipleMetrics(this)
multiMetrics.input = inputBam multiMetrics.input = inputBam
multiMetrics.outputName = new File(outputDir, inputBam.getName.stripSuffix(".bam")) multiMetrics.outputName = new File(outputDir, inputBam.getName.stripSuffix(".bam"))
if (!paired)
multiMetrics.program = multiMetrics.program
.filter(_ != CollectMultipleMetrics.Programs.CollectInsertSizeMetrics)
add(multiMetrics) add(multiMetrics)
addSummarizable(multiMetrics, "multi_metrics") addSummarizable(multiMetrics, "multi_metrics")
...@@ -129,13 +138,19 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -129,13 +138,19 @@ class BamMetrics(val parent: Configurable) extends QScript
ampBedToInterval.isIntermediate = true ampBedToInterval.isIntermediate = true
add(ampBedToInterval) add(ampBedToInterval)
val chsMetrics = CollectHsMetrics(this, inputBam, val chsMetrics = CollectHsMetrics(this,
List(ampIntervals), ampIntervals :: roiIntervals.map(_.intervals), outputDir) inputBam,
List(ampIntervals),
ampIntervals :: roiIntervals.map(_.intervals),
outputDir)
add(chsMetrics) add(chsMetrics)
addSummarizable(chsMetrics, "hs_metrics") addSummarizable(chsMetrics, "hs_metrics")
val pcrMetrics = CollectTargetedPcrMetrics(this, inputBam, val pcrMetrics = CollectTargetedPcrMetrics(this,
ampIntervals, ampIntervals :: roiIntervals.map(_.intervals), outputDir) inputBam,
ampIntervals,
ampIntervals :: roiIntervals.map(_.intervals),
outputDir)
add(pcrMetrics) add(pcrMetrics)
addSummarizable(pcrMetrics, "targeted_pcr_metrics") addSummarizable(pcrMetrics, "targeted_pcr_metrics")
...@@ -145,36 +160,48 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -145,36 +160,48 @@ class BamMetrics(val parent: Configurable) extends QScript
// Create stats and coverage plot for each bed/interval file // Create stats and coverage plot for each bed/interval file
val allIntervalNames = (roiIntervals ++ ampIntervals).map(_.bed.getName) val allIntervalNames = (roiIntervals ++ ampIntervals).map(_.bed.getName)
if (allIntervalNames.size != allIntervalNames.toSet.size) { if (allIntervalNames.size != allIntervalNames.toSet.size) {
logger.warn("There are multiple region files with the same name. Metric values might get overwritten") logger.warn(
"There are multiple region files with the same name. Metric values might get overwritten")
} }
for (intervals <- roiIntervals ++ ampIntervals) { for (intervals <- roiIntervals ++ ampIntervals) {
val targetName = intervals.bed.getName.stripSuffix(".bed") val targetName = intervals.bed.getName.stripSuffix(".bed")
val targetDir = new File(outputDir, targetName) val targetDir = new File(outputDir, targetName)
val biStrict = BedtoolsIntersect(this, inputBam, intervals.bed, val biStrict = BedtoolsIntersect(
this,
inputBam,
intervals.bed,
output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.strict.sam"), output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.strict.sam"),
minOverlap = config("strict_intersect_overlap", default = 1.0)) minOverlap = config("strict_intersect_overlap", default = 1.0)
)
val biopetFlagstatStrict = BiopetFlagstat(this, biStrict.output, targetDir) val biopetFlagstatStrict = BiopetFlagstat(this, biStrict.output, targetDir)
addSummarizable(biopetFlagstatStrict, targetName + "_flagstats_strict") addSummarizable(biopetFlagstatStrict, targetName + "_flagstats_strict")
add(new BiopetFifoPipe(this, List(biStrict, biopetFlagstatStrict))) add(new BiopetFifoPipe(this, List(biStrict, biopetFlagstatStrict)))
val biLoose = BedtoolsIntersect(this, inputBam, intervals.bed, val biLoose = BedtoolsIntersect(
this,
inputBam,
intervals.bed,
output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.loose.sam"), output = new File(targetDir, inputBam.getName.stripSuffix(".bam") + ".overlap.loose.sam"),
minOverlap = config("loose_intersect_overlap", default = 0.01)) minOverlap = config("loose_intersect_overlap", default = 0.01)
)
val biopetFlagstatLoose = BiopetFlagstat(this, biLoose.output, targetDir) val biopetFlagstatLoose = BiopetFlagstat(this, biLoose.output, targetDir)
addSummarizable(biopetFlagstatLoose, targetName + "_flagstats_loose") addSummarizable(biopetFlagstatLoose, targetName + "_flagstats_loose")
add(new BiopetFifoPipe(this, List(biLoose, biopetFlagstatLoose))) add(new BiopetFifoPipe(this, List(biLoose, biopetFlagstatLoose)))
val sortedBed = BamMetrics.sortedbedCache.getOrElse(intervals.bed, { val sortedBed = BamMetrics.sortedbedCache.getOrElse(
intervals.bed, {
val sorter = new BedtoolsSort(this) val sorter = new BedtoolsSort(this)
sorter.input = intervals.bed sorter.input = intervals.bed
sorter.output = swapExt(targetDir, intervals.bed, ".bed", ".sorted.bed") sorter.output = swapExt(targetDir, intervals.bed, ".bed", ".sorted.bed")
add(sorter) add(sorter)
BamMetrics.sortedbedCache += intervals.bed -> sorter.output BamMetrics.sortedbedCache += intervals.bed -> sorter.output
sorter.output sorter.output
}) }
)
val bedCov = BedtoolsCoverage(this, sortedBed, inputBam, depth = true) val bedCov = BedtoolsCoverage(this, sortedBed, inputBam, depth = true)
val covStats = CoverageStats(this, targetDir, inputBam.getName.stripSuffix(".bam") + ".coverage") val covStats =
CoverageStats(this, targetDir, inputBam.getName.stripSuffix(".bam") + ".coverage")
covStats.title = Some("Coverage Plot") covStats.title = Some("Coverage Plot")
covStats.subTitle = Some(s"for file '$targetName.bed'") covStats.subTitle = Some(s"for file '$targetName.bed'")
add(bedCov | covStats) add(bedCov | covStats)
...@@ -186,9 +213,12 @@ class BamMetrics(val parent: Configurable) extends QScript ...@@ -186,9 +213,12 @@ class BamMetrics(val parent: Configurable) extends QScript
} }
object BamMetrics extends PipelineCommand { object BamMetrics extends PipelineCommand {
/** Make default implementation of BamMetrics and runs script already */ /** Make default implementation of BamMetrics and runs script already */
def apply(root: Configurable, def apply(root: Configurable,
bamFile: File, outputDir: File, bamFile: File,
outputDir: File,
paired: Boolean,
sampleId: Option[String] = None, sampleId: Option[String] = None,
libId: Option[String] = None): BamMetrics = { libId: Option[String] = None): BamMetrics = {
val bamMetrics = new BamMetrics(root) val bamMetrics = new BamMetrics(root)
...@@ -196,6 +226,7 @@ object BamMetrics extends PipelineCommand { ...@@ -196,6 +226,7 @@ object BamMetrics extends PipelineCommand {
bamMetrics.libId = libId bamMetrics.libId = libId
bamMetrics.inputBam = bamFile bamMetrics.inputBam = bamFile
bamMetrics.outputDir = outputDir bamMetrics.outputDir = outputDir
bamMetrics.paired = paired
bamMetrics.init() bamMetrics.init()