Commit be7838f2 authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub

Merge pull request #93 from biopet/release-0.9.0

Release 0.9.0
parents 26269612 70d48352
...@@ -6,33 +6,45 @@ node('local') { ...@@ -6,33 +6,45 @@ node('local') {
tool 'Apache Maven 3.3.9' tool 'Apache Maven 3.3.9'
} }
timeout(45) {
stage('Checkout') { stage('Checkout') {
checkout scm checkout scm
sh 'git submodule update --init --recursive' sh 'git submodule update --init --recursive'
} }
}
timeout(45) {
stage('Build and Test') { stage('Build and Test') {
withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') { withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package' sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
} }
} }
}
stage('Report tests') { stage('Report Tests') {
junit '*/target/surefire-reports/*.xml' junit '*/target/surefire-reports/*.xml'
} }
if(currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) { stage('Check git on changes') {
sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
}
stage('Check Documentation') {
sh 'mkdocs build --clean --strict'
}
if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
currentBuild.result = "SUCCESS" currentBuild.result = "SUCCESS"
slackSend (color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc') slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else { } else {
slackSend (color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc') slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} }
} catch (e) { } catch (e) {
if(currentBuild.result == null || "FAILED".equals(currentBuild.result)) { if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
currentBuild.result = "FAILED" currentBuild.result = "FAILED"
} }
slackSend (color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc') slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
throw e throw e
} }
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
Biopet (Bio Pipeline Execution Toolkit) is the main pipeline development framework of the LUMC Sequencing Analysis Support Core team. It contains our main pipelines and some of the command line tools we develop in-house. It is meant to be used in the main [SHARK](https://humgenprojects.lumc.nl/trac/shark) computing cluster. While usage outside of SHARK is technically possible, some adjustments may need to be made in order to do so. Biopet (Bio Pipeline Execution Toolkit) is the main pipeline development framework of the LUMC Sequencing Analysis Support Core team. It contains our main pipelines and some of the command line tools we develop in-house. It is meant to be used in the main [SHARK](https://humgenprojects.lumc.nl/trac/shark) computing cluster. While usage outside of SHARK is technically possible, some adjustments may need to be made in order to do so.
Full documantation is here: [Biopet documantation](http://biopet-docs.readthedocs.io/en/latest/) Full documentation is here: [Biopet documentation](http://biopet-docs.readthedocs.io/en/latest/)
## Quick Start ## Quick Start
...@@ -60,7 +60,7 @@ Biopet is based on the Queue framework developed by the Broad Institute as part ...@@ -60,7 +60,7 @@ Biopet is based on the Queue framework developed by the Broad Institute as part
We welcome any kind of contribution, be it merge requests on the code base, documentation updates, or any kinds of other fixes! The main language we use is Scala, though the repository also contains a small bit of Python and R. Our main code repository is located at [https://github.com/biopet/biopet](https://github.com/biopet/biopet/issues), along with our issue tracker. We welcome any kind of contribution, be it merge requests on the code base, documentation updates, or any kinds of other fixes! The main language we use is Scala, though the repository also contains a small bit of Python and R. Our main code repository is located at [https://github.com/biopet/biopet](https://github.com/biopet/biopet/issues), along with our issue tracker.
For more information please go to our [Developer documantation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/) For more information please go to our [Developer documentation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/)
## About ## About
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
<parent> <parent>
<groupId>nl.lumc.sasc</groupId> <groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId> <artifactId>Biopet</artifactId>
<version>0.8.0</version> <version>0.9.0</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
...@@ -44,18 +44,6 @@ ...@@ -44,18 +44,6 @@
<artifactId>BiopetExtensions</artifactId> <artifactId>BiopetExtensions</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.8</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>
\ No newline at end of file
...@@ -27,7 +27,7 @@ import org.broadinstitute.gatk.queue.QScript ...@@ -27,7 +27,7 @@ import org.broadinstitute.gatk.queue.QScript
* *
* Created by pjvan_thof on 1/29/15. * Created by pjvan_thof on 1/29/15.
*/ */
class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript { class Bam2Wig(val parent: Configurable) extends QScript with BiopetQScript {
def this() = this(null) def this() = this(null)
@Input(doc = "Input bam file", required = true) @Input(doc = "Input bam file", required = true)
......
...@@ -28,7 +28,7 @@ import scala.collection.JavaConversions._ ...@@ -28,7 +28,7 @@ import scala.collection.JavaConversions._
* *
* Created by pjvan_thof on 1/29/15. * Created by pjvan_thof on 1/29/15.
*/ */
class BamToChromSizes(val root: Configurable) extends InProcessFunction with Configurable { class BamToChromSizes(val parent: Configurable) extends InProcessFunction with Configurable {
@Input @Input
var bamFile: File = _ var bamFile: File = _
......
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
<parent> <parent>
<groupId>nl.lumc.sasc</groupId> <groupId>nl.lumc.sasc</groupId>
<artifactId>Biopet</artifactId> <artifactId>Biopet</artifactId>
<version>0.8.0</version> <version>0.9.0</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
...@@ -42,17 +42,5 @@ ...@@ -42,17 +42,5 @@
<artifactId>BiopetToolsExtensions</artifactId> <artifactId>BiopetToolsExtensions</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.2.1</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>
#import(nl.lumc.sasc.biopet.utils.summary.Summary) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage) #import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport) #import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File) #import(java.io.File)
<%@ var summary: Summary %> <%@ var summary: SummaryDb %>
<%@ var sampleId: Option[String] = None %> <%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[String] = None %> <%@ var libId: Option[Int] = None %>
<%@ var sampleLevel: Boolean = false %> <%@ var sampleLevel: Boolean = false %>
<%@ var rootPath: String %> <%@ var rootPath: String %>
<%@ var outputDir: File %> <%@ var outputDir: File %>
<%@ var showPlot: Boolean = false %> <%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %> <%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%> <%@ var showIntro: Boolean = true%>
<%@ var runId: Int %>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(sample) => { case Some(id) => allSamples.filter(_.id == id).toList
List(sample.toString) case _ => allSamples.toList
}
case _ => summary.samples.toList
} }
}# }#
#if (showIntro) #if (showIntro)
...@@ -69,24 +75,31 @@ ...@@ -69,24 +75,31 @@
<th>(%)</th> <th>(%)</th>
</tr></thead> </tr></thead>
<tbody> <tbody>
#for (sample <- samples.toList.sorted) #for (sample <- samples.sortBy(_.name))
#{ #{
val libs = (libId, sampleLevel) match { val libs: List[Option[Int]] = (libId, sampleLevel) match {
case (_, true) => List("") case (_, true) => List(None)
case (Some(libId), _) => List(libId.toString).sorted case (Some(_), _) => List(libId)
case _ => summary.libraries(sample).toList.sorted case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList
} }
}# }#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td> <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
#for (libId <- libs) #for (libId <- libs)
#{ val libName = libId.map(l => allLibraries.find(_.id == l).get.name) }#
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats") val statsPaths = Map(
val total = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "All")):_*).getOrElse(0L).asInstanceOf[Long] "All" -> List("flagstats", "All"),
val mapped = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long] "Mapped" -> List("flagstats", "Mapped"),
val duplicates = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long] "Duplicates" -> List("flagstats", "Duplicates"),
val secondary = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "NotPrimaryAlignment")):_*).getOrElse(0L).asInstanceOf[Long] "NotPrimaryAlignment" -> List("flagstats", "NotPrimaryAlignment")
)
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
val total = results("All").getOrElse(0L).asInstanceOf[Long]
val mapped = results("Mapped").getOrElse(0L).asInstanceOf[Long]
val duplicates = results("Duplicates").getOrElse(0L).asInstanceOf[Long]
val secondary = results("NotPrimaryAlignment").getOrElse(0L).asInstanceOf[Long]
}# }#
<td>${total}</td> <td>${total}</td>
<td>${mapped}</td> <td>${mapped}</td>
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Run)
#import(nl.lumc.sasc.biopet.core.report.ReportPage) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Sample)
<%@ var summary: Summary %> #import(nl.lumc.sasc.biopet.utils.summary.db.Schema.Library)
<%@ var rootPath: String %> <%@ var runId: Int %>
<%@ var sampleId: Option[String] %> <%@ var sampleId: Option[Int] = None %>
<%@ var libId: Option[String] = None %> <%@ var libId: Option[Int] = None %>
<%@ var run: Run %>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
<table class="table"> <table class="table">
<tbody> <tbody>
<tr><th>Pipeline</th><td>BamMetrics</td></tr> <tr><th>Pipeline</th><td>GearsSingle</td></tr>
<tr><th>Version</th><td>${summary.getValue("meta", "pipeline_version")}</td></tr> <tr><th>Version</th><td>${run.version}</td></tr>
<tr><th>Last commit hash</th><td>${summary.getValue("meta", "last_commit_hash")}</td></tr> <tr><th>Last commit hash</th><td>${run.commitHash}</td></tr>
<tr><th>Output directory</th><td>${summary.getValue("meta", "output_dir")}</td></tr> <tr><th>Output directory</th><td>${run.outputDir}</td></tr>
<tr><th>Sample ID</th><td>${sampleId}</td></tr> <tr><th>Sample</th><td>${allSamples.filter(_.id == sampleId.get).headOption.map(_.name)}</td></tr>
#if (libId.isDefined) <tr><th>Library ID</th><td>${libId}</td></tr> #end <tr><th>Library</th><td>${allLibraries.filter(_.id == libId.get).headOption.map(_.name)}</td></tr>
</tbody> </tbody>
</table> </table>
<br/> <br/>
......
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var fields: List[String] = List("All", "Mapped", "Duplicates", "MAPQ>30", "MateUnmapped", "Mate on other chr")%>
<table>
<tbody>
#for (field <- fields)
<tr><th>${field}</th><td>
#if (libId.isDefined)
${summary.getLibraryValue(sampleId.get, libId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#else
${summary.getSampleValue(sampleId.get, metricsTag, "stats", "bamstats", "flagstats", field)}
#end
</td></tr>
#end
</tbody>
</table>
#import(nl.lumc.sasc.biopet.utils.summary.Summary)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(java.io.File)
<%@ var summary: Summary %>
<%@ var sampleId: Option[String] %>
<%@ var libId: Option[String] = None %>
<%@ var metricsTag: String = "bammetrics" %>
<table class="table sortable-theme-bootstrap">
<thead><tr>
<th>Path</th>
<th>MD5</th>
</tr></thead>
<tbody>
<tr>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "path")}</td>
<td>${summary.getValue(sampleId, libId, metricsTag, "files", "pipeline", "bamfile", "md5")}</td>
</tr>
</tbody>
</table>
\ No newline at end of file
#import(nl.lumc.sasc.biopet.utils.summary.Summary) #import(nl.lumc.sasc.biopet.utils.summary.db.Schema._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
#import(nl.lumc.sasc.biopet.core.report.ReportPage) #import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport) #import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
#import(java.io.File) #import(java.io.File)
#import(org.apache.commons.io.FileUtils) <%@ var summary: SummaryDb %>
<%@ var summary: Summary %> <%@ var sampleId: Option[Int] = None %>
<%@ var sampleId: Option[String] = None %> <%@ var libId: Option[Int] = None %>
<%@ var libId: Option[String] = None %>
<%@ var rootPath: String %>
<%@ var metricsTag: String = "bammetrics" %>
<%@ var sampleLevel: Boolean = false %> <%@ var sampleLevel: Boolean = false %>
<%@ var rootPath: String %>
<%@ var outputDir: File %> <%@ var outputDir: File %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var showPlot: Boolean = false %> <%@ var showPlot: Boolean = false %>
<%@ var showTable: Boolean = true %> <%@ var showTable: Boolean = true %>
<%@ var showIntro: Boolean = true%> <%@ var showIntro: Boolean = true%>
<%@ var runId: Int %>
<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
<%@ var allSamples: Seq[Sample] %>
<%@ var allLibraries: Seq[Library] %>
#{ #{
val samples = sampleId match { val samples = sampleId match {
case Some(sample) => { case Some(id) => allSamples.filter(_.id == id).toList
List(sample.toString) case _ => allSamples.toList
}
case _ => summary.samples.toList
} }
}# }#
#if (showIntro) #if (showIntro)
<br/> <br/>
<div class="row"> <div class="row">
...@@ -42,7 +43,7 @@ ...@@ -42,7 +43,7 @@
#end #end
#if (showPlot) #if (showPlot)
#{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libId = libId) }# #{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }#
<div class="panel-body"> <div class="panel-body">
<img src="clipping.png" class="img-responsive" /> <img src="clipping.png" class="img-responsive" />
...@@ -71,27 +72,25 @@ ...@@ -71,27 +72,25 @@
#end #end
</tr></thead> </tr></thead>
<tbody> <tbody>
#for (sample <- samples.toList.sorted) #for (sample <- samples.sortBy(_.name))
#{ #{
val libs = (libId, sampleLevel) match { val libs: List[Option[Int]] = (libId, sampleLevel) match {
case (_, true) => List("") case (_, true) => List(None)
case (Some(libId), _) => List(libId.toString) case (Some(_), _) => List(libId)
case _ => summary.libraries(sample).toList case _ => allLibraries.filter(_.sampleId == sample.id).map(x => Some(x.id)).toList
} }
}# }#
<tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td> <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
#for (libId <- libs) #for (libId <- libs)
#{ val libName = libId.map(l => allLibraries.find(_.id == l).get.name) }#
#if (libs.head != libId) <tr> #end #if (libs.head != libId) <tr> #end
#if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
#{ #{
val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats") val statsPaths = fields.map(x => x -> List("clipping", "general", x)).toMap
val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
val fieldValues = for (field <- fields) yield {
summary.getValue((prefixPath ::: List("bamstats", "clipping", "general", field)):_*).getOrElse("N/A")
}
}# }#
#for (value <- fieldValues) #for (field <- fields)
<td>${value}</td> <td>${results(field)}</td>
#end #end
</tr> </tr>
#end #end
......