Commit 3e35c32b authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub

Merge branch 'develop' into fix-BIOPET-635

parents 172d242a 625bb514
node('local') {
timeout(45) {
try {
try {
stage('Init') {
tool 'JDK 8u102'
tool 'Apache Maven 3.3.9'
}
stage('Init') {
tool 'JDK 8u102'
tool 'Apache Maven 3.3.9'
}
timeout(45) {
stage('Checkout') {
checkout scm
sh 'git submodule update --init --recursive'
}
}
timeout(45) {
stage('Build and Test') {
withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
}
}
}
stage('Report Tests') {
junit '*/target/surefire-reports/*.xml'
}
stage('Check git on changes') {
sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
}
stage('Report Tests') {
junit '*/target/surefire-reports/*.xml'
}
stage('Check Documentation') {
sh 'mkdocs build --clean --strict'
}
stage('Check git on changes') {
sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
}
if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
currentBuild.result = "SUCCESS"
slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else {
slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
}
stage('Check Documentation') {
sh 'mkdocs build --clean --strict'
}
} catch (e) {
if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
currentBuild.result = "FAILED"
}
slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
currentBuild.result = "SUCCESS"
slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
} else {
slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
}
throw e
} catch (e) {
if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
currentBuild.result = "FAILED"
}
slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
throw e
}
}
......@@ -24,8 +24,7 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Await
import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration
class BammetricsReport(val parent: Configurable) extends ReportBuilderExtension {
......@@ -42,28 +41,27 @@ object BammetricsReport extends ReportBuilder {
/** Name of report */
val reportName = "Bam Metrics"
def pipelineName = "bammetrics"
/** Root page for single BamMetrcis report */
def indexPage = {
val bamMetricsPage = this.bamMetricsPage(summary, sampleId, libId)
ReportPage(bamMetricsPage.subPages ::: List(
"Versions" -> ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
)), Map()),
"Files" -> ReportPage(List(), List(), Map())
), List(
"Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
) ::: bamMetricsPage.sections,
Map()
)
}
def indexPage: Future[ReportPage] =
bamMetricsPage(summary, sampleId, libId).map { bamMetricsPage =>
ReportPage(bamMetricsPage.subPages ::: List(
"Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
)), Map())),
"Files" -> filesPage(sampleId, libId)
), List(
"Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
) ::: bamMetricsPage.sections,
Map()
)
}
/** Generates a page with alignment stats */
def bamMetricsPage(summary: SummaryDb,
sampleId: Option[Int],
libId: Option[Int],
metricsTag: String = "bammetrics") = {
//val pipelineId: Int = summary.getPipelineId(runId, metricsTag).map(_.get)
metricsTag: String = "bammetrics"): Future[ReportPage] = Future {
val wgsExecuted = summary.getStatsSize(runId, metricsTag, "wgs", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
val rnaExecuted = summary.getStatsSize(runId, metricsTag, "rna", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
......@@ -85,10 +83,10 @@ object BammetricsReport extends ReportBuilder {
ReportPage(
if (targets.isEmpty) List()
else List("Targets" -> ReportPage(
else List("Targets" -> Future.successful(ReportPage(
List(),
targets.map(t => t -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp", Map("target" -> Some(t)))),
Map())),
Map()))),
List(
"Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"),
"Mapping Quality" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/mappingQuality.ssp", Map("showPlot" -> true)),
......@@ -161,7 +159,7 @@ object BammetricsReport extends ReportBuilder {
plot.output = pngFile
plot.ylabel = Some("Reads")
plot.width = Some(200 + (results.size * 10))
plot.title = Some("Aligned reads")
plot.title = Some("Aligned_reads")
plot.runLocal()
}
......
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration)
#import(scala.concurrent.Future)
<%@ var summary: SummaryDb %>
<%@ var indexPage: ReportPage %>
<%@ var indexPage: Future[ReportPage] %>
<%@ var reportName: String %>
<%@ var page: ReportPage %>
<%@ var path: List[String] %>
<%@ var args: Map[String, Any] %>
<%@ var rootPath: String %>
#{
val iPage = Await.result(indexPage, Duration.Inf)
def createMenu(page: ReportPage, path: List[String] = Nil, first: Boolean = true): String = {
val buffer: StringBuffer = new StringBuffer()
......@@ -15,11 +20,13 @@
buffer.append("<ul class=\"dropdown-menu list-group\">")
}
for (subPage <- page.subPages.sortBy(_._1)) {
for (subPageFuture <- page.subPages.sortBy(_._1)) {
val subPageName = subPageFuture._1
val subPage = Await.result(subPageFuture._2, Duration.Inf)
val href: String = {
if (path.isEmpty) rootPath + subPage._1 + "/index.html"
else rootPath + path.mkString("","/","/") + subPage._1 + "/index.html"
if (path.isEmpty) rootPath + subPageName + "/index.html"
else rootPath + path.mkString("","/","/") + subPageName + "/index.html"
}
// buffer.append("<li")
......@@ -30,16 +37,16 @@
// buffer.append(createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false))
// buffer.append("</li>")
val listSubmenu = if(subPage._2.subPages.nonEmpty) "dropdown-submenu" else ""
val listSubmenu = if(subPage.subPages.nonEmpty) "dropdown-submenu" else ""
// val subMenuBadgeCount = if(subPage._2.subPages.nonEmpty && first) "<span class='badge'>%d</span>".format(subPage._2.subPages.size) else ""
val tabIndex = if (first) " tabindex='-1'" else ""
// val listGroupA = if(subPage._2.subPages.nonEmpty) "list-group-item" else ""
var menuItem: String = "<li class='%s'>".format(listSubmenu) +
"<a href='%s' class='%s'%s>".format(href, "", tabIndex) +
"%s".format(subPage._1) +
"%s".format(subPageName) +
"</a>" +
createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false) +
createMenu(subPage, path ::: subPageName :: Nil, first = false) +
"</li>"
buffer.append(menuItem)
......@@ -51,10 +58,10 @@
}
def getSubPage(path:List[String]): ReportPage = {
path.foldLeft(indexPage)((c, p) => {
path.foldLeft(iPage)((c, p) => {
val foundPages = c.subPages.filter(_._1 == p)
require(foundPages.size == 1, "Name of page not found or duplicate is found, page:: " + p + " in path: " + path)
foundPages.head._2
Await.result(foundPages.head._2, Duration.Inf)
})
}
}#
......@@ -111,9 +118,9 @@
<ul class="nav navbar-nav">
<li class="root #if (path.isEmpty) active #end">
<a class="navbar-brand" href="${rootPath}index.html">${reportName}
#if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
#if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
</a>
${unescape(createMenu(indexPage))}
${unescape(createMenu(iPage))}
</li>
</ul>
</div>
......@@ -126,9 +133,9 @@
#if(t == 0)
<li class="root">
<a href="${rootPath}index.html">Home
#if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
#if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
</a>
${unescape(createMenu(indexPage, Nil, false))}
${unescape(createMenu(iPage, Nil, false))}
</li>
#else
<li class="root #if (t == path.size) active #end">
......
......@@ -14,7 +14,7 @@
*/
package nl.lumc.sasc.biopet.core.report
import scala.concurrent.Await
import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration
/**
......@@ -25,46 +25,44 @@ import scala.concurrent.duration.Duration
trait MultisampleReportBuilder extends ReportBuilder {
/** Method to generate a single sample page */
def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage
def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage]
/** Default list of samples, can be override */
def samplesSections: List[(String, ReportSection)] = {
List(
("Samples", ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
)
List("Samples" -> ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
}
/** Method to generate a single library page */
def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): ReportPage
def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): Future[ReportPage]
/** Default list of libraries, can be override */
def librariesSections: List[(String, ReportSection)] = {
List(
("Libraries", ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
)
List("Libraries" -> ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
}
/** Generate the samples page including a single sample page for each sample in the summary */
def generateSamplesPage(args: Map[String, Any]): ReportPage = {
val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
val samplePages = samples.map(_.id)
.map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId))))
.toList
ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
}
def generateSamplesPage(args: Map[String, Any]): Future[ReportPage] =
summary.getSamples(runId = Some(runId)).map { samples =>
val samplePages = samples.map(_.id)
.map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId)))
.map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, None) :: Nil)))
.toList
ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
}
/** Generate the libraries page for a single sample with a subpage for eacht library */
def generateLibraryPage(args: Map[String, Any]): ReportPage = {
def generateLibraryPage(args: Map[String, Any]): Future[ReportPage] = {
val sampleId = args("sampleId") match {
case Some(x: Int) => x
case None => throw new IllegalStateException("Sample not found")
}
val libraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)), Duration.Inf)
val libPages = libraries.map(_.id)
.map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId))))
.toList
ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)).map { libraries =>
val libPages = libraries.map(_.id)
.map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId)))
.map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, libId) :: Nil)))
.toList
ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
}
}
}
......@@ -14,6 +14,8 @@
*/
package nl.lumc.sasc.biopet.core.report
import scala.concurrent.Future
/**
* Created by pjvan_thof on 3/27/15.
*
......@@ -21,6 +23,6 @@ package nl.lumc.sasc.biopet.core.report
* @param sections Sections for this page
* @param args Arguments for this page, this arguments get passed to all section and subpages
*/
case class ReportPage(subPages: List[(String, ReportPage)],
case class ReportPage(subPages: List[(String, Future[ReportPage])],
sections: List[(String, ReportSection)],
args: Map[String, Any])
......@@ -26,6 +26,7 @@ import nl.lumc.sasc.biopet.LastCommitHash
import scala.collection.mutable
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
import scala.io.Source
/**
......
......@@ -16,6 +16,7 @@ package nl.lumc.sasc.biopet.core.report
import java.io.File
import java.nio.file.Paths
import java.sql.Date
import com.google.common.io.Files
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
......@@ -23,7 +24,7 @@ import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test
import scala.concurrent.Await
import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration
/**
......@@ -37,19 +38,26 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
@Test
def testGeneratePages(): Unit = {
val builder = new MultisampleReportBuilder {
def pipelineName = "test"
def reportName: String = "test"
def indexPage: ReportPage = ReportPage("Samples" -> generateSamplesPage(Map()) :: Nil, Nil, Map())
def indexPage: Future[ReportPage] = Future(ReportPage("Samples" -> generateSamplesPage(Map()) :: Nil, Nil, Map()))
def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage =
ReportPage("Libraries" -> generateLibraryPage(Map("sampleId" -> Some(sampleId))) :: Nil, Nil, Map())
def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage] =
Future(ReportPage("Libraries" -> generateLibraryPage(Map("sampleId" -> Some(sampleId))) :: Nil, Nil, Map()))
def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = ReportPage(Nil, Nil, Map())
def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = Future(ReportPage(Nil, Nil, Map()))
}
import scala.concurrent.ExecutionContext.Implicits.global
val dbFile = File.createTempFile("summary.", ".db")
dbFile.deleteOnExit()
val db = SummaryDb.openSqliteSummary(dbFile)
db.createTables()
Await.ready(db.createPipeline("test", 0), Duration.Inf)
Await.ready(db.createRun("test", "", "", "", new Date(System.currentTimeMillis())), Duration.Inf)
val sample = Some("sampleName")
val lib = Some("libName")
......
......@@ -16,6 +16,7 @@ package nl.lumc.sasc.biopet.core.report
import java.io.File
import java.nio.file.Paths
import java.sql.Date
import com.google.common.io.Files
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
......@@ -23,7 +24,7 @@ import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.{ DataProvider, Test }
import scala.concurrent.Await
import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration
/**
......@@ -46,11 +47,14 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
@Test(dataProvider = "testGeneratePages")
def testGeneratePages(sample: Option[String], lib: Option[String], nested: Boolean): Unit = {
val builder = new ReportBuilder {
def pipelineName = "test"
def reportName: String = "test"
def indexPage: ReportPage = ReportPage(
(if (nested) "p1" -> ReportPage(Nil, Nil, Map()) :: Nil else Nil), Nil, Map())
def indexPage: Future[ReportPage] = Future(ReportPage(
(if (nested) "p1" -> Future(ReportPage(Nil, Nil, Map())) :: Nil else Nil), Nil, Map()))
}
import scala.concurrent.ExecutionContext.Implicits.global
val dbFile = File.createTempFile("summary.", ".db")
dbFile.deleteOnExit()
val db = SummaryDb.openSqliteSummary(dbFile)
......@@ -61,6 +65,8 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
Await.result(db.createLibrary(libName, 0, sampleId), Duration.Inf)
}
}
Await.ready(db.createPipeline("test", 0), Duration.Inf)
Await.ready(db.createRun("test", "", "", "", new Date(System.currentTimeMillis())), Duration.Inf)
val tempDir = Files.createTempDir()
tempDir.deleteOnExit()
......@@ -77,22 +83,22 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
db.close()
}
@Test
def testCountPages: Unit = {
ReportBuilder.countPages(ReportPage(Nil, Nil, Map())) shouldBe 1
ReportBuilder.countPages(ReportPage(
"p1" -> ReportPage(Nil, Nil, Map()) :: Nil,
Nil, Map())) shouldBe 2
ReportBuilder.countPages(ReportPage(
"p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage(Nil, Nil, Map()) :: Nil,
Nil, Map())) shouldBe 3
ReportBuilder.countPages(ReportPage(
"p1" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
Nil, Map())) shouldBe 3
ReportBuilder.countPages(ReportPage(
"p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
Nil, Map())) shouldBe 4
}
// @Test
// def testCountPages: Unit = {
// ReportBuilder.countPages(ReportPage(Nil, Nil, Map())) shouldBe 1
// ReportBuilder.countPages(ReportPage(
// "p1" -> ReportPage(Nil, Nil, Map()) :: Nil,
// Nil, Map())) shouldBe 2
// ReportBuilder.countPages(ReportPage(
// "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage(Nil, Nil, Map()) :: Nil,
// Nil, Map())) shouldBe 3
// ReportBuilder.countPages(ReportPage(
// "p1" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
// Nil, Map())) shouldBe 3
// ReportBuilder.countPages(ReportPage(
// "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
// Nil, Map())) shouldBe 4
// }
@Test
def testRenderTemplate: Unit = {
......
......@@ -32,6 +32,7 @@ import org.testng.annotations.{ AfterClass, Test }
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.matching.Regex
/**
......
......@@ -14,7 +14,7 @@
*/
package nl.lumc.sasc.biopet.tools
import java.io.{ PrintWriter, InputStream, File }
import java.io.{ File, PrintWriter }
import java.util
import htsjdk.variant.vcf.VCFFileReader
......@@ -24,6 +24,8 @@ import nl.lumc.sasc.biopet.utils.intervals.{ BedRecord, BedRecordList }
import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.concurrent.ExecutionContext
import scala.concurrent.ExecutionContext.Implicits.global
object RegionAfCount extends ToolCommand {
case class Args(bedFile: File = null,
......@@ -123,7 +125,7 @@ object RegionAfCount extends ToolCommand {
if (cmdArgs.scatterpPlot) generatePlot(tsvFile)
}
def generatePlot(tsvFile: File): Unit = {
def generatePlot(tsvFile: File)(implicit ec: ExecutionContext): Unit = {
logger.info(s"Generate plot for $tsvFile")
val scatterPlot = new ScatterPlot(null)
......
......@@ -20,6 +20,8 @@ import nl.lumc.sasc.biopet.utils.rscript.LinePlot
import nl.lumc.sasc.biopet.utils.{ Logging, sortAnyAny }
import scala.collection.mutable
import scala.concurrent.ExecutionContext
import scala.concurrent.ExecutionContext.Implicits.global
/**
* Created by pjvanthof on 05/07/16.
......@@ -91,7 +93,7 @@ class Histogram[T](_counts: Map[T, Long] = Map[T, Long]())(implicit ord: Numeric
writer.close()
}
def writeFilesAndPlot(outputDir: File, prefix: String, xlabel: String, ylabel: String, title: String): Unit = {
def writeFilesAndPlot(outputDir: File, prefix: String, xlabel: String, ylabel: String, title: String)(implicit ec: ExecutionContext): Unit = {
writeHistogramToTsv(new File(outputDir, prefix + ".histogram.tsv"))
writeAggregateToTsv(new File(outputDir, prefix + ".stats.tsv"))
val plot = new LinePlot(null)
......
package nl.lumc.sasc.biopet.utils.process
/*
The MIT License (MIT)
Copyright (c) 2016 j-keck <jhyphenkeck@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
import nl.lumc.sasc.biopet.utils.Logging
import scala.collection.parallel.mutable.ParMap
import scala.concurrent.duration.Duration
import scala.concurrent._
import scala.sys.process.{ Process, ProcessLogger }
import scala.util.Try
object Sys extends Sys {
type ExitValue = Int
type Stdout = String
type Stderr = String
type ExecResult = (ExitValue, Stdout, Stderr)
trait AsyncExecResult {
/**
* @see [[scala.concurrent.Future#map]]
*/
def map[T](f: ExecResult => T): Future[T]
/**
* @see [[scala.concurrent.Future#foreach]]
*/
def foreach(f: ExecResult => Unit): Unit
/**
* @see [[scala.concurrent.Future#onComplete]]
*/
def onComplete[T](pf: Try[ExecResult] => T): Unit
/**
* cancels the running process
*/
def cancel(): Unit
/**
* check if the process is still running
* @return `true` if the process is already completed, `false` otherwise
*/
def isRunning: Boolean
/**
* the underlying future
* @return the future, in which the process runs
*/
def get: Future[ExecResult]
}
type Cancelable = () => Unit
case class ExecutionCanceled(msg: String) extends Exception(msg)
}
trait Sys {
import Sys._
private val cache: ParMap[Seq[String], AsyncExecResult] = ParMap()
var maxRunningProcesses: Int = 5
def exec(cmd: String): ExecResult = exec(cmd.split(" "))
/**
* executes the cmd and blocks until the command exits.
*
* @return {{{(ExitValue, Stdout, Stderr)}}}
* <pre>if the executable is unable to start, (-1, "", stderr) are returned</pre>
*/
def exec(cmd: Seq[String]): ExecResult = {
val stdout = new OutputSlurper
val stderr = new OutputSlurper
Try {
val proc = Process(cmd).run(ProcessLogger(stdout.appendLine, stderr.appendLine))
proc.exitValue()
}.map((_, stdout.get, stderr.get))
.recover {
case t => (-1, "", t.getMessage)
}.get
}
def execAsync(cmd: String)(implicit ec: ExecutionContext): AsyncExecResult = execAsync(cmd.split(" "))(ec)
/**
* executes the cmd asynchronous
* @see scala.concurrent.Future.map
*
* @return [[AsyncExecResult]]
*/
def execAsync(cmd: Seq[String])(implicit ec: ExecutionContext): AsyncExecResult = {
while (cache.size >= maxRunningProcesses) {
for ((cmd, c) <- cache.toList) {
val results = Option(c)
if (!results.map(_.isRunning).getOrElse(true)) try {
cache -= cmd
} catch {
case e: NullPointerException =>
}
else try {
results.foreach(x => Await.ready(x.get, Duration.fromNanos(100000)))
} catch {
case e: TimeoutException =>
}
}
}
val results = new AsyncExecResult {
val (fut, cancelFut) = runAsync(cmd)
override def map[T](f: ExecResult => T): Future[T] = fut.map(f)
override def foreach(f: ExecResult => Unit): Unit = fut.foreach(f)