Commit be7838f2 authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub

Merge pull request #93 from biopet/release-0.9.0

Release 0.9.0
parents 26269612 70d48352
......@@ -6,33 +6,45 @@ node('local') {
tool 'Apache Maven 3.3.9'
}
stage('Checkout') {
checkout scm
sh 'git submodule update --init --recursive'
timeout(45) {
stage('Checkout') {
checkout scm
sh 'git submodule update --init --recursive'
}
}
stage('Build and Test') {
withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
timeout(45) {
stage('Build and Test') {
withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
}
}
}
stage('Report tests') {
stage('Report Tests') {
junit '*/target/surefire-reports/*.xml'
}
if(currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
stage('Check git on changes') {
sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
}
stage('Check Documentation') {
sh 'mkdocs build --clean --strict'
}
if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
currentBuild.result = "SUCCESS"
slackSend (color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else {
slackSend (color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
}
} catch (e) {
if(currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
currentBuild.result = "FAILED"
}
slackSend (color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
throw e
}
......
......@@ -5,7 +5,7 @@
Biopet (Bio Pipeline Execution Toolkit) is the main pipeline development framework of the LUMC Sequencing Analysis Support Core team. It contains our main pipelines and some of the command line tools we develop in-house. It is meant to be used in the main [SHARK](https://humgenprojects.lumc.nl/trac/shark) computing cluster. While usage outside of SHARK is technically possible, some adjustments may need to be made in order to do so.
Full documantation is here: [Biopet documantation](http://biopet-docs.readthedocs.io/en/latest/)
Full documentation is here: [Biopet documentation](http://biopet-docs.readthedocs.io/en/latest/)
## Quick Start
......@@ -60,7 +60,7 @@ Biopet is based on the Queue framework developed by the Broad Institute as part
We welcome any kind of contribution, be it merge requests on the code base, documentation updates, or any kinds of other fixes! The main language we use is Scala, though the repository also contains a small bit of Python and R. Our main code repository is located at [https://github.com/biopet/biopet](https://github.com/biopet/biopet/issues), along with our issue tracker.
For more information please go to our [Developer documantation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/)
For more information please go to our [Developer documentation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/)
## About
......
......@@ -26,7 +26,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.8.0</version>
<version>0.9.0</version>
<relativePath>../</relativePath>
</parent>
......@@ -44,18 +44,6 @@
<artifactId>BiopetExtensions</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.8</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
......@@ -27,7 +27,7 @@ import org.broadinstitute.gatk.queue.QScript
*
* Created by pjvan_thof on 1/29/15.
*/
class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript {
class Bam2Wig(val parent: Configurable) extends QScript with BiopetQScript {
def this() = this(null)
@Input(doc = "Input bam file", required = true)
......
......@@ -28,7 +28,7 @@ import scala.collection.JavaConversions._
*
* Created by pjvan_thof on 1/29/15.
*/
class BamToChromSizes(val root: Configurable) extends InProcessFunction with Configurable {
class BamToChromSizes(val parent: Configurable) extends InProcessFunction with Configurable {
@Input
var bamFile: File = _
......
......@@ -24,7 +24,7 @@
<parent>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId>
<version>0.8.0</version>
<version>0.9.0</version>
<relativePath>../</relativePath>
</parent>
......@@ -42,17 +42,5 @@
<artifactId>BiopetToolsExtensions</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var summary: SummaryDb %>
<%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[Int] = None %>
<%@ var sampleLevel: Boolean = false %>
<%@ var rootPath: String %>
<%@ var outputDir: File %>
<%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%>
<%@ var runId: Int %>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
#{
val samples = sampleId match {
case Some(sample) => {
List(sample.toString)
}
case _ => summary.samples.toList
case Some(id) => allSamples.filter(_.id == id).toList
case _ => allSamples.toList
}
}#
#if (showIntro)
......@@ -69,24 +75,31 @@
<th>(%)</th>
</tr></thead>
<tbody>
#for (sample <- samples.toList.sorted)
#for (sample <- samples.sortBy(_.name))
#{
val libs = (libId, sampleLevel) match {
case (_, true) => List("")
case (Some(libId), _) => List(libId.toString).sorted
case _ => summary.libraries(sample).toList.sorted
val libs: List[Option[Int]] = (libId, sampleLevel) match {
case (_, true) => List(None)
case (Some(_), _) => List(libId)
case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList
}
}#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
#for (libId <- libs)
#{ val libName = libId.map(l => allLibraries.find(_.id == l).get.name) }#
#if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
val total = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "All")):_*).getOrElse(0L).asInstanceOf[Long]
val mapped = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long]
val duplicates = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long]
val secondary = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "NotPrimaryAlignment")):_*).getOrElse(0L).asInstanceOf[Long]
val statsPaths = Map(
"All" -> List("flagstats", "All"),
"Mapped" -> List("flagstats", "Mapped"),
"Duplicates" -> List("flagstats", "Duplicates"),
"NotPrimaryAlignment" -> List("flagstats", "NotPrimaryAlignment")
)
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
val total = results("All").getOrElse(0L).asInstanceOf[Long]
val mapped = results("Mapped").getOrElse(0L).asInstanceOf[Long]
val duplicates = results("Duplicates").getOrElse(0L).asInstanceOf[Long]
val secondary = results("NotPrimaryAlignment").getOrElse(0L).asInstanceOf[Long]
}#
<td>${total}</td>
<td>${mapped}</td>
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
<%@ var summary: Summary %>
<%@ var rootPath: String %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Run)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
<%@ var runId: Int %>
<%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[Int] = None %>
<%@ var run: Run %>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
<table class="table">
<tbody>
<tr><th>Pipeline</th><td>BamMetrics</td></tr>
<tr><th>Version</th><td>${summary.getValue("meta", "pipeline_version")}</td></tr>
<tr><th>Last commit hash</th><td>${summary.getValue("meta", "last_commit_hash")}</td></tr>
<tr><th>Output directory</th><td>${summary.getValue("meta", "output_dir")}</td></tr>
<tr><th>Sample ID</th><td>${sampleId}</td></tr>
#if (libId.isDefined) <tr><th>Library ID</th><td>${libId}</td></tr> #end
<tr><th>Pipeline</th><td>GearsSingle</td></tr>
<tr><th>Version</th><td>${run.version}</td></tr>
<tr><th>Last commit hash</th><td>${run.commitHash}</td></tr>
<tr><th>Output directory</th><td>${run.outputDir}</td></tr>
<tr><th>Sample</th><td>${allSamples.filter(_.id == sampleId.get).headOption.map(_.name)}</td></tr>
<tr><th>Library</th><td>${allLibraries.filter(_.id == libId.get).headOption.map(_.name)}</td></tr>
</tbody>
</table>
<br/>
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var fields: List[String] = List("All", "Mapped", "Duplicates", "MAPQ>30", "MateUnmapped", "Mate on other chr")%>
<table>
<tbody>
#for (field <- fields)
<tr><th>${field}</th><td>
#if (libId.isDefined)
${summary.getLibraryValue(sampleId.get, libId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#else
${summary.getSampleValue(sampleId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#end
</td></tr>
#end
</tbody>
</table>
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(java.io.File)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
<%@ var metricsTag: String = "bammetrics" %>
<table class="table sortable-theme-bootstrap">
<thead><tr>
<th>Path</th>
<th>MD5</th>
</tr></thead>
<tbody>
<tr>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "path")}</td>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "md5")}</td>
</tr>
</tbody>
</table>
\ No newline at end of file
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File)
#import(org.apache.commons.io.FileUtils)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var summary: SummaryDb %>
<%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[Int] = None %>
<%@ var sampleLevel: Boolean = false %>
<%@ var rootPath: String %>
<%@ var outputDir: File %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%>
<%@ var runId: Int %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
#{
val samples = sampleId match {
case Some(sample) => {
List(sample.toString)
}
case _ => summary.samples.toList
case Some(id) => allSamples.filter(_.id == id).toList
case _ => allSamples.toList
}
}#
#if (showIntro)
<br/>
<div class="row">
......@@ -42,7 +43,7 @@
#end
#if (showPlot)
#{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libId = libId) }#
#{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }#
<div class="panel-body">
<img src="clipping.png" class="img-responsive" />
......@@ -71,27 +72,25 @@
#end
</tr></thead>
<tbody>
#for (sample <- samples.toList.sorted)
#for (sample <- samples.sortBy(_.name))
#{
val libs = (libId, sampleLevel) match {
case (_, true) => List("")
case (Some(libId), _) => List(libId.toString)
case _ => summary.libraries(sample).toList
val libs: List[Option[Int]] = (libId, sampleLevel) match {
case (_, true) => List(None)
case (Some(_), _) => List(libId)
case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList
}
}#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
#for (libId <- libs)
#{ val libName = libId.map(l => allLibraries.find(_.id == l).get.name) }#
#if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
val fieldValues = for (field <- fields) yield {
summary.getValue((prefixPath ::: List("bamstats", "clipping", "general", field)):_*).getOrElse("N/A")
}
val statsPaths = fields.map(x => x -> List("clipping", "general", x)).toMap
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
}#
#for (value <- fieldValues)
<td>${value}</td>
#for (field <- fields)
<td>${results(field)}</td>
#end
</tr>
#end
......
#import(nl.lumc.sasc.biopet.utils.IoUtils)
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(org.apache.commons.io.FileUtils)
#import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] = None %>
<%@ var libId: Option[String] = None %>
<%@ var summary: SummaryDb %>
<%@ var sampleId: Option[Int] = None %>
<%@ var sampleLevel: Boolean = true %>
<%@ var rootPath: String %>
<%@ var libId: Option[Int] = None %>
<%@ var outputDir: File %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var target: Option[String] %>
<%@ var runId: Int %>
<%@ var fields: List[String] = List("mean", "median", "max", "horizontal", "frac_min_10x", "frac_min_20x", "frac_min_30x", "frac_min_40x", "frac_min_50x") %>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
#{
val samples = sampleId match {
case Some(sample) => List(sample.toString)
case _ => summary.samples.toList
case Some(id) => allSamples.filter(_.id == id).toList
case _ => allSamples.toList
}
}#
<table class="table">
<thead><tr>
<th>sample</th>
<th>mean</th>
<th>median</th>
<th>max</th>
<th>horizontal</th>
<th>frac min 10x</th>
<th>frac min 20x</th>
<th>frac min 30x</th>
<th>frac min 40x</th>
<th>frac min 50x</th>
#for (field <- fields)
<th>${field}</th>
#end
</tr></thead>
<tbody>
#for (sample <- samples.toList.sorted)
#for (sample <- samples.sortBy(_.name))
#{
val libs = (libId, sampleLevel) match {
case (_, true) => List("")
case (Some(libId), _) => List(libId.toString)
case _ => summary.libraries(sample).toList
val libs: List[Option[Int]] = (libId, sampleLevel) match {
case (_, true) => List(None)
case (Some(_), _) => List(libId)
case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList