Commit c6b8bddc authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

WIP

parent dea3adbb
...@@ -160,7 +160,7 @@ object BammetricsReport extends ReportBuilder { ...@@ -160,7 +160,7 @@ object BammetricsReport extends ReportBuilder {
plot.output = pngFile plot.output = pngFile
plot.ylabel = Some("Reads") plot.ylabel = Some("Reads")
plot.width = Some(200 + (results.size * 10)) plot.width = Some(200 + (results.size * 10))
plot.title = Some("Aligned reads") plot.title = Some("Aligned_reads")
plot.runLocal() plot.runLocal()
} }
......
...@@ -16,7 +16,6 @@ package nl.lumc.sasc.biopet.core.report ...@@ -16,7 +16,6 @@ package nl.lumc.sasc.biopet.core.report
import scala.concurrent.{ Await, Future } import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* This trait will generate a report with added function to generate sample and library pages for those existing in the summary. * This trait will generate a report with added function to generate sample and library pages for those existing in the summary.
......
...@@ -17,14 +17,14 @@ package nl.lumc.sasc.biopet.core.report ...@@ -17,14 +17,14 @@ package nl.lumc.sasc.biopet.core.report
import java.io._ import java.io._
import nl.lumc.sasc.biopet.core.ToolCommandFunction import nl.lumc.sasc.biopet.core.ToolCommandFunction
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample } import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Sample}
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand } import nl.lumc.sasc.biopet.utils.{IoUtils, Logging, ToolCommand}
import org.broadinstitute.gatk.utils.commandline.Input import org.broadinstitute.gatk.utils.commandline.Input
import org.fusesource.scalate.TemplateEngine import org.fusesource.scalate.TemplateEngine
import scala.collection.mutable import scala.collection.mutable
import scala.concurrent.{ Await, ExecutionContextExecutor, Future } import scala.concurrent._
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.language.postfixOps import scala.language.postfixOps
import scala.language.implicitConversions import scala.language.implicitConversions
...@@ -73,7 +73,7 @@ trait ReportBuilderExtension extends ToolCommandFunction { ...@@ -73,7 +73,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
trait ReportBuilder extends ToolCommand { trait ReportBuilder extends ToolCommand {
implicit lazy val global: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global implicit lazy val ec = ReportBuilder.ec
implicit def toOption[T](x: T): Option[T] = Option(x) implicit def toOption[T](x: T): Option[T] = Option(x)
implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf) implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
...@@ -278,6 +278,8 @@ trait ReportBuilder extends ToolCommand { ...@@ -278,6 +278,8 @@ trait ReportBuilder extends ToolCommand {
object ReportBuilder { object ReportBuilder {
implicit lazy val ec = ExecutionContext.global
/** Single template render engine, this will have a cache for all compile templates */ /** Single template render engine, this will have a cache for all compile templates */
protected val engine = new TemplateEngine() protected val engine = new TemplateEngine()
engine.allowReload = false engine.allowReload = false
......
...@@ -26,6 +26,7 @@ import nl.lumc.sasc.biopet.LastCommitHash ...@@ -26,6 +26,7 @@ import nl.lumc.sasc.biopet.LastCommitHash
import scala.collection.mutable import scala.collection.mutable
import scala.concurrent.Await import scala.concurrent.Await
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
import scala.io.Source import scala.io.Source
/** /**
......
...@@ -25,7 +25,6 @@ import org.testng.annotations.Test ...@@ -25,7 +25,6 @@ import org.testng.annotations.Test
import scala.concurrent.{ Await, Future } import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* Created by pjvanthof on 24/02/16. * Created by pjvanthof on 24/02/16.
...@@ -46,6 +45,9 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers { ...@@ -46,6 +45,9 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = Future(ReportPage(Nil, Nil, Map())) def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = Future(ReportPage(Nil, Nil, Map()))
} }
import scala.concurrent.ExecutionContext.Implicits.global
val dbFile = File.createTempFile("summary.", ".db") val dbFile = File.createTempFile("summary.", ".db")
dbFile.deleteOnExit() dbFile.deleteOnExit()
val db = SummaryDb.openSqliteSummary(dbFile) val db = SummaryDb.openSqliteSummary(dbFile)
......
...@@ -26,8 +26,6 @@ import org.testng.annotations.{ DataProvider, Test } ...@@ -26,8 +26,6 @@ import org.testng.annotations.{ DataProvider, Test }
import scala.concurrent.{ Await, Future } import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* Created by pjvanthof on 24/02/16. * Created by pjvanthof on 24/02/16.
*/ */
...@@ -53,6 +51,8 @@ class ReportBuilderTest extends TestNGSuite with Matchers { ...@@ -53,6 +51,8 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
(if (nested) "p1" -> Future(ReportPage(Nil, Nil, Map())) :: Nil else Nil), Nil, Map())) (if (nested) "p1" -> Future(ReportPage(Nil, Nil, Map())) :: Nil else Nil), Nil, Map()))
} }
import scala.concurrent.ExecutionContext.Implicits.global
val dbFile = File.createTempFile("summary.", ".db") val dbFile = File.createTempFile("summary.", ".db")
dbFile.deleteOnExit() dbFile.deleteOnExit()
val db = SummaryDb.openSqliteSummary(dbFile) val db = SummaryDb.openSqliteSummary(dbFile)
......
...@@ -32,6 +32,7 @@ import org.testng.annotations.{ AfterClass, Test } ...@@ -32,6 +32,7 @@ import org.testng.annotations.{ AfterClass, Test }
import scala.concurrent.Await import scala.concurrent.Await
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.matching.Regex import scala.util.matching.Regex
/** /**
......
...@@ -14,16 +14,18 @@ ...@@ -14,16 +14,18 @@
*/ */
package nl.lumc.sasc.biopet.tools package nl.lumc.sasc.biopet.tools
import java.io.{ PrintWriter, InputStream, File } import java.io.{File, PrintWriter}
import java.util import java.util
import htsjdk.variant.vcf.VCFFileReader import htsjdk.variant.vcf.VCFFileReader
import nl.lumc.sasc.biopet.utils.ToolCommand import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.rscript.ScatterPlot import nl.lumc.sasc.biopet.utils.rscript.ScatterPlot
import nl.lumc.sasc.biopet.utils.intervals.{ BedRecord, BedRecordList } import nl.lumc.sasc.biopet.utils.intervals.{BedRecord, BedRecordList}
import scala.collection.JavaConversions._ import scala.collection.JavaConversions._
import scala.collection.mutable import scala.collection.mutable
import scala.concurrent.ExecutionContext
import scala.concurrent.ExecutionContext.Implicits.global
object RegionAfCount extends ToolCommand { object RegionAfCount extends ToolCommand {
case class Args(bedFile: File = null, case class Args(bedFile: File = null,
...@@ -123,7 +125,7 @@ object RegionAfCount extends ToolCommand { ...@@ -123,7 +125,7 @@ object RegionAfCount extends ToolCommand {
if (cmdArgs.scatterpPlot) generatePlot(tsvFile) if (cmdArgs.scatterpPlot) generatePlot(tsvFile)
} }
def generatePlot(tsvFile: File): Unit = { def generatePlot(tsvFile: File)(implicit ec: ExecutionContext): Unit = {
logger.info(s"Generate plot for $tsvFile") logger.info(s"Generate plot for $tsvFile")
val scatterPlot = new ScatterPlot(null) val scatterPlot = new ScatterPlot(null)
......
...@@ -14,12 +14,14 @@ ...@@ -14,12 +14,14 @@
*/ */
package nl.lumc.sasc.biopet.tools.bamstats package nl.lumc.sasc.biopet.tools.bamstats
import java.io.{ File, IOException, PrintWriter } import java.io.{File, IOException, PrintWriter}
import nl.lumc.sasc.biopet.utils.rscript.LinePlot import nl.lumc.sasc.biopet.utils.rscript.LinePlot
import nl.lumc.sasc.biopet.utils.{ Logging, sortAnyAny } import nl.lumc.sasc.biopet.utils.{Logging, sortAnyAny}
import scala.collection.mutable import scala.collection.mutable
import scala.concurrent.ExecutionContext
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* Created by pjvanthof on 05/07/16. * Created by pjvanthof on 05/07/16.
...@@ -91,7 +93,7 @@ class Histogram[T](_counts: Map[T, Long] = Map[T, Long]())(implicit ord: Numeric ...@@ -91,7 +93,7 @@ class Histogram[T](_counts: Map[T, Long] = Map[T, Long]())(implicit ord: Numeric
writer.close() writer.close()
} }
def writeFilesAndPlot(outputDir: File, prefix: String, xlabel: String, ylabel: String, title: String): Unit = { def writeFilesAndPlot(outputDir: File, prefix: String, xlabel: String, ylabel: String, title: String)(implicit ec: ExecutionContext): Unit = {
writeHistogramToTsv(new File(outputDir, prefix + ".histogram.tsv")) writeHistogramToTsv(new File(outputDir, prefix + ".histogram.tsv"))
writeAggregateToTsv(new File(outputDir, prefix + ".stats.tsv")) writeAggregateToTsv(new File(outputDir, prefix + ".stats.tsv"))
val plot = new LinePlot(null) val plot = new LinePlot(null)
......
...@@ -17,6 +17,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ...@@ -17,6 +17,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ */
import nl.lumc.sasc.biopet.utils.Logging
import scala.concurrent.{ ExecutionContext, Future, Promise } import scala.concurrent.{ ExecutionContext, Future, Promise }
import scala.sys.process.{ Process, ProcessLogger } import scala.sys.process.{ Process, ProcessLogger }
import scala.util.Try import scala.util.Try
...@@ -129,12 +131,16 @@ trait Sys { ...@@ -129,12 +131,16 @@ trait Sys {
val proc = Process(cmd).run(ProcessLogger(stdout.appendLine, stderr.appendLine)) val proc = Process(cmd).run(ProcessLogger(stdout.appendLine, stderr.appendLine))
p.tryCompleteWith(Future(proc.exitValue).map(c => (c, stdout.get, stderr.get))) p.tryCompleteWith(Future(proc.exitValue).map(c => (c, stdout.get, stderr.get)))
val cancel = { val cancel = () => {
p.tryFailure(new ExecutionCanceled(s"Process: '${cmd.mkString(" ")}' canceled")) p.tryFailure {
Logging.logger.error("stdout: " + stdout.get)
Logging.logger.error("stderr: " + stderr.get)
new ExecutionCanceled(s"Process: '${cmd.mkString(" ")}' canceled")
}
proc.destroy() proc.destroy()
} }
(p.future, () => cancel) (p.future, cancel)
} }
class OutputSlurper { class OutputSlurper {
......
...@@ -14,12 +14,16 @@ ...@@ -14,12 +14,16 @@
*/ */
package nl.lumc.sasc.biopet.utils.rscript package nl.lumc.sasc.biopet.utils.rscript
import java.io.{ File, FileOutputStream } import java.io.{File, FileOutputStream}
import nl.lumc.sasc.biopet.utils.Logging import nl.lumc.sasc.biopet.utils.Logging
import nl.lumc.sasc.biopet.utils.config.Configurable import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.process.Sys import nl.lumc.sasc.biopet.utils.process.Sys
import nl.lumc.sasc.biopet.utils.process.Sys.AsyncExecResult
import scala.collection.parallel.mutable.ParSeq
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext}
import scala.sys.process.ProcessLogger import scala.sys.process.ProcessLogger
/** /**
...@@ -62,16 +66,26 @@ trait Rscript extends Configurable { ...@@ -62,16 +66,26 @@ trait Rscript extends Configurable {
} }
} }
private val cache: ParSeq[AsyncExecResult] = Nil
/** /**
* Execute rscript on local system * Execute rscript on local system
* @param logger How to handle stdout and stderr * @param logger How to handle stdout and stderr
*/ */
def runLocal(logger: ProcessLogger): Unit = { def runLocal(logger: ProcessLogger)(implicit ec: ExecutionContext): Unit = {
checkScript() checkScript()
Logging.logger.info("Running: " + cmd.mkString(" ")) Logging.logger.info("Running: " + cmd.mkString(" "))
val (exitcode, stdout, stderr) = Sys.exec(cmd) while (cache.size > 5) for (c <- cache) Thread.
val results = Sys.execAsync(cmd)
while(results.isRunning) {
Thread.sleep(1000)
}
val (exitcode, stdout, stderr) = Await.result(results.map(x => (x._1, x._2, x._3)), Duration.Inf)
Logging.logger.info("stdout:\n" + stdout + "\n") Logging.logger.info("stdout:\n" + stdout + "\n")
Logging.logger.info("stderr:\n" + stderr) Logging.logger.info("stderr:\n" + stderr)
...@@ -83,7 +97,7 @@ trait Rscript extends Configurable { ...@@ -83,7 +97,7 @@ trait Rscript extends Configurable {
* Execute rscript on local system * Execute rscript on local system
* Stdout and stderr will go to biopet logger * Stdout and stderr will go to biopet logger
*/ */
def runLocal(): Unit = { def runLocal()(implicit ec: ExecutionContext): Unit = {
runLocal(ProcessLogger(Logging.logger.info(_))) runLocal(ProcessLogger(Logging.logger.info(_)))
} }
} }
...@@ -4,7 +4,6 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils ...@@ -4,7 +4,6 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils
import nl.lumc.sasc.biopet.utils.summary.db.Schema._ import nl.lumc.sasc.biopet.utils.summary.db.Schema._
import slick.driver.H2Driver.api._ import slick.driver.H2Driver.api._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.{ Await, ExecutionContext, Future } import scala.concurrent.{ Await, ExecutionContext, Future }
import java.io.{ Closeable, File } import java.io.{ Closeable, File }
...@@ -22,6 +21,8 @@ import scala.language.implicitConversions ...@@ -22,6 +21,8 @@ import scala.language.implicitConversions
*/ */
trait SummaryDb extends Closeable { trait SummaryDb extends Closeable {
implicit val ec: ExecutionContext
def db: Database def db: Database
def close(): Unit = db.close() def close(): Unit = db.close()
...@@ -431,9 +432,9 @@ trait SummaryDb extends Closeable { ...@@ -431,9 +432,9 @@ trait SummaryDb extends Closeable {
} }
class SummaryDbReadOnly(val db: Database) extends SummaryDb class SummaryDbReadOnly(val db: Database)(implicit val ec: ExecutionContext) extends SummaryDb
class SummaryDbWrite(val db: Database) extends SummaryDb { class SummaryDbWrite(val db: Database)(implicit val ec: ExecutionContext) extends SummaryDb {
/** This method will create all tables */ /** This method will create all tables */
def createTables(): Unit = { def createTables(): Unit = {
try { try {
...@@ -688,7 +689,7 @@ object SummaryDb { ...@@ -688,7 +689,7 @@ object SummaryDb {
} }
/** This will open a sqlite database and create tables when the database did not exist yet */ /** This will open a sqlite database and create tables when the database did not exist yet */
def openSqliteSummary(file: File): SummaryDbWrite = { def openSqliteSummary(file: File)(implicit ec: ExecutionContext): SummaryDbWrite = {
if (!summaryConnections.contains(file)) { if (!summaryConnections.contains(file)) {
val config: org.sqlite.SQLiteConfig = new org.sqlite.SQLiteConfig() val config: org.sqlite.SQLiteConfig = new org.sqlite.SQLiteConfig()
config.enforceForeignKeys(true) config.enforceForeignKeys(true)
...@@ -703,7 +704,7 @@ object SummaryDb { ...@@ -703,7 +704,7 @@ object SummaryDb {
summaryConnections(file) summaryConnections(file)
} }
def openReadOnlySqliteSummary(file: File): SummaryDbReadOnly = { def openReadOnlySqliteSummary(file: File)(implicit ec: ExecutionContext): SummaryDbReadOnly = {
require(file.exists(), s"File does not exist: $file") require(file.exists(), s"File does not exist: $file")
val config: org.sqlite.SQLiteConfig = new org.sqlite.SQLiteConfig() val config: org.sqlite.SQLiteConfig = new org.sqlite.SQLiteConfig()
config.enforceForeignKeys(true) config.enforceForeignKeys(true)
...@@ -712,7 +713,7 @@ object SummaryDb { ...@@ -712,7 +713,7 @@ object SummaryDb {
config.setReadOnly(true) config.setReadOnly(true)
val asyncExecutor = new AsyncExecutor { val asyncExecutor = new AsyncExecutor {
override def executionContext: ExecutionContext = global override def executionContext: ExecutionContext = ec
override def close(): Unit = {} override def close(): Unit = {}
} }
......
...@@ -9,6 +9,8 @@ import org.testng.annotations.Test ...@@ -9,6 +9,8 @@ import org.testng.annotations.Test
import scala.concurrent.Await import scala.concurrent.Await
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
import SummaryDb.Implicts._ import SummaryDb.Implicts._
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoModule, NoSample } import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoModule, NoSample }
......
...@@ -24,7 +24,6 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoModule, Sam ...@@ -24,7 +24,6 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoModule, Sam
import scala.concurrent.{ Await, Future } import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* Report for Gears * Report for Gears
......
...@@ -23,7 +23,6 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._ ...@@ -23,7 +23,6 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._
import scala.concurrent.{ Await, Future } import scala.concurrent.{ Await, Future }
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global
/** /**
* Created by pjvanthof on 11/01/16. * Created by pjvanthof on 11/01/16.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment