Merge branch 'develop' into CleverFixVCF

bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala

@@ -24,8 +24,7 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._
 
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 
 class BammetricsReport(val parent: Configurable) extends ReportBuilderExtension {
@@ -42,28 +41,27 @@ object BammetricsReport extends ReportBuilder {
   /** Name of report */
   val reportName = "Bam Metrics"
 
+  def pipelineName = "bammetrics"
+
   /** Root page for single BamMetrcis report */
-  def indexPage = {
-    val bamMetricsPage = this.bamMetricsPage(summary, sampleId, libId)
-    ReportPage(bamMetricsPage.subPages ::: List(
-      "Versions" -> ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
-      )), Map()),
-      "Files" -> ReportPage(List(), List(), Map())
-    ), List(
-      "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
-    ) ::: bamMetricsPage.sections,
-      Map()
-    )
-  }
+  def indexPage: Future[ReportPage] =
+    bamMetricsPage(summary, sampleId, libId).map { bamMetricsPage =>
+      ReportPage(bamMetricsPage.subPages ::: List(
+        "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
+        )), Map())),
+        "Files" -> filesPage(sampleId, libId)
+      ), List(
+        "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
+      ) ::: bamMetricsPage.sections,
+        Map()
+      )
+    }
 
   /** Generates a page with alignment stats */
   def bamMetricsPage(summary: SummaryDb,
                      sampleId: Option[Int],
                      libId: Option[Int],
-                     metricsTag: String = "bammetrics") = {
-
-    //val pipelineId: Int = summary.getPipelineId(runId, metricsTag).map(_.get)
-
+                     metricsTag: String = "bammetrics"): Future[ReportPage] = Future {
     val wgsExecuted = summary.getStatsSize(runId, metricsTag, "wgs", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
     val rnaExecuted = summary.getStatsSize(runId, metricsTag, "rna", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
 
@@ -85,10 +83,10 @@ object BammetricsReport extends ReportBuilder {
 
     ReportPage(
       if (targets.isEmpty) List()
-      else List("Targets" -> ReportPage(
+      else List("Targets" -> Future.successful(ReportPage(
         List(),
         targets.map(t => t -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp", Map("target" -> Some(t)))),
-        Map())),
+        Map()))),
       List(
         "Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"),
         "Mapping Quality" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/mappingQuality.ssp", Map("showPlot" -> true)),
@@ -161,7 +159,7 @@ object BammetricsReport extends ReportBuilder {
     plot.output = pngFile
     plot.ylabel = Some("Reads")
     plot.width = Some(200 + (results.size * 10))
-    plot.title = Some("Aligned reads")
+    plot.title = Some("Aligned_reads")
     plot.runLocal()
   }
 

biopet-core/src/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp

 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
 #import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
+#import(scala.concurrent.Future)
 <%@ var summary: SummaryDb %>
-<%@ var indexPage: ReportPage %>
+<%@ var indexPage: Future[ReportPage] %>
 <%@ var reportName: String %>
 <%@ var page: ReportPage %>
 <%@ var path: List[String] %>
 <%@ var args: Map[String, Any] %>
 <%@ var rootPath: String %>
 #{
+    val iPage = Await.result(indexPage, Duration.Inf)
+
     def createMenu(page: ReportPage, path: List[String] = Nil, first: Boolean = true): String = {
         val buffer: StringBuffer = new StringBuffer()
 
@@ -15,11 +20,13 @@
             buffer.append("<ul class=\"dropdown-menu list-group\">")
         }
 
-        for (subPage <- page.subPages.sortBy(_._1)) {
+        for (subPageFuture <- page.subPages.sortBy(_._1)) {
 
+            val subPageName = subPageFuture._1
+            val subPage = Await.result(subPageFuture._2, Duration.Inf)
             val href: String = {
-                if (path.isEmpty) rootPath + subPage._1 + "/index.html"
-                else rootPath + path.mkString("","/","/") + subPage._1 + "/index.html"
+                if (path.isEmpty) rootPath + subPageName + "/index.html"
+                else rootPath + path.mkString("","/","/") + subPageName + "/index.html"
             }
 
 //            buffer.append("<li")
@@ -30,16 +37,16 @@
 //            buffer.append(createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false))
 //            buffer.append("</li>")
 
-            val listSubmenu = if(subPage._2.subPages.nonEmpty) "dropdown-submenu" else ""
+            val listSubmenu = if(subPage.subPages.nonEmpty) "dropdown-submenu" else ""
 //            val subMenuBadgeCount = if(subPage._2.subPages.nonEmpty && first) "<span class='badge'>%d</span>".format(subPage._2.subPages.size) else ""
             val tabIndex = if (first) " tabindex='-1'" else ""
 //            val listGroupA = if(subPage._2.subPages.nonEmpty) "list-group-item" else ""
 
             var menuItem: String = "<li class='%s'>".format(listSubmenu) +
             "<a href='%s' class='%s'%s>".format(href, "", tabIndex) +
-            "%s".format(subPage._1) +
+            "%s".format(subPageName) +
             "</a>" +
-            createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false) +
+            createMenu(subPage, path ::: subPageName :: Nil, first = false) +
             "</li>"
             buffer.append(menuItem)
 
@@ -51,10 +58,10 @@
     }
 
     def getSubPage(path:List[String]): ReportPage = {
-        path.foldLeft(indexPage)((c, p) => {
+        path.foldLeft(iPage)((c, p) => {
             val foundPages = c.subPages.filter(_._1 == p)
             require(foundPages.size == 1, "Name of page not found or duplicate is found, page:: " + p + " in path: " + path)
-            foundPages.head._2
+            Await.result(foundPages.head._2, Duration.Inf)
         })
     }
 }#
@@ -111,9 +118,9 @@
         <ul class="nav navbar-nav">
             <li class="root #if (path.isEmpty) active #end">
                 <a class="navbar-brand" href="${rootPath}index.html">${reportName}
-                    #if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
+                    #if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
                 </a>
-                ${unescape(createMenu(indexPage))}
+                ${unescape(createMenu(iPage))}
             </li>
         </ul>
     </div>
@@ -126,9 +133,9 @@
                 #if(t == 0)
                     <li class="root">
                         <a href="${rootPath}index.html">Home
-                            #if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
+                            #if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
                         </a>
-                        ${unescape(createMenu(indexPage, Nil, false))}
+                        ${unescape(createMenu(iPage, Nil, false))}
                     </li>
                 #else
                     <li class="root #if (t == path.size) active #end">

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetCommandLineFunction.scala

@@ -99,7 +99,7 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
     beforeGraph()
     internalBeforeGraph()
 
-    this.commandDirectory = this.jobOutputFile.getParentFile
+    if (jobOutputFile != null) this.commandDirectory = this.jobOutputFile.getAbsoluteFile.getParentFile
 
     super.freezeFieldValues()
   }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala

@@ -14,7 +14,7 @@
  */
 package nl.lumc.sasc.biopet.core.report
 
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 
 /**
@@ -25,46 +25,44 @@ import scala.concurrent.duration.Duration
 trait MultisampleReportBuilder extends ReportBuilder {
 
   /** Method to generate a single sample page */
-  def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage
+  def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of samples, can be override */
   def samplesSections: List[(String, ReportSection)] = {
-    List(
-      ("Samples", ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
-    )
+    List("Samples" -> ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
   }
 
   /** Method to generate a single library page */
-  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): ReportPage
+  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of libraries, can be override */
   def librariesSections: List[(String, ReportSection)] = {
-    List(
-      ("Libraries", ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
-    )
+    List("Libraries" -> ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
   }
 
   /** Generate the samples page including a single sample page for each sample in the summary */
-  def generateSamplesPage(args: Map[String, Any]): ReportPage = {
-    val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
-    val samplePages = samples.map(_.id)
-      .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId))))
-      .toList
-    ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
-  }
+  def generateSamplesPage(args: Map[String, Any]): Future[ReportPage] =
+    summary.getSamples(runId = Some(runId)).map { samples =>
+      val samplePages = samples.map(_.id)
+        .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId)))
+          .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, None) :: Nil)))
+        .toList
+      ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
+    }
 
   /** Generate the libraries page for a single sample with a subpage for eacht library */
-  def generateLibraryPage(args: Map[String, Any]): ReportPage = {
+  def generateLibraryPage(args: Map[String, Any]): Future[ReportPage] = {
     val sampleId = args("sampleId") match {
       case Some(x: Int) => x
       case None         => throw new IllegalStateException("Sample not found")
     }
 
-    val libraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)), Duration.Inf)
-
-    val libPages = libraries.map(_.id)
-      .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId))))
-      .toList
-    ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
+    summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)).map { libraries =>
+      val libPages = libraries.map(_.id)
+        .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId)))
+          .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, libId) :: Nil)))
+        .toList
+      ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
+    }
   }
 }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -17,14 +17,15 @@ package nl.lumc.sasc.biopet.core.report
 import java.io._
 
 import nl.lumc.sasc.biopet.core.ToolCommandFunction
-import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
+import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample, Run }
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ LibraryId, SampleId }
 import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
 import org.broadinstitute.gatk.utils.commandline.Input
 import org.fusesource.scalate.TemplateEngine
 
 import scala.collection.mutable
-import scala.concurrent.{ Await, Future }
+import scala.concurrent._
 import scala.concurrent.duration.Duration
 import scala.language.postfixOps
 import scala.language.implicitConversions
@@ -39,7 +40,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
   /** Report builder object */
   def builder: ReportBuilder
 
-  def toolObject = builder
+  def toolObject: ReportBuilder = builder
 
   @Input(required = true)
   var summaryDbFile: File = _
@@ -73,6 +74,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
 
 trait ReportBuilder extends ToolCommand {
 
+  implicit lazy val ec = ReportBuilder.ec
   implicit def toOption[T](x: T): Option[T] = Option(x)
   implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
 
@@ -112,20 +114,24 @@ trait ReportBuilder extends ToolCommand {
   private var setSummary: SummaryDb = _
 
   /** Retrival of summary, read only */
-  final def summary = setSummary
+  final def summary: SummaryDb = setSummary
 
   private var setRunId: Int = 0
 
-  final def runId = setRunId
+  final def runId: Int = setRunId
+
+  private var _setRun: Run = _
+
+  final def run: Run = _setRun
 
   private var _setPipelines = Seq[Pipeline]()
-  final def pipelines = _setPipelines
+  final def pipelines: Seq[Pipeline] = _setPipelines
   private var _setModules = Seq[Module]()
-  final def modules = _setModules
+  final def modules: Seq[Module] = _setModules
   private var _setSamples = Seq[Sample]()
-  final def samples = _setSamples
+  final def samples: Seq[Sample] = _setSamples
   private var _setLibraries = Seq[Library]()
-  final def libraries = _setLibraries
+  final def libraries: Seq[Library] = _setLibraries
 
   /** default args that are passed to all page withing the report */
   def pageArgs: Map[String, Any] = Map()
@@ -134,13 +140,13 @@ trait ReportBuilder extends ToolCommand {
   private var total = 0
 
   private var _sampleId: Option[Int] = None
-  protected[report] def sampleId = _sampleId
+  protected[report] def sampleId: Option[Int] = _sampleId
   private var _libId: Option[Int] = None
-  protected[report] def libId = _libId
+  protected[report] def libId: Option[Int] = _libId
 
   case class ExtFile(resourcePath: String, targetPath: String)
 
-  def extFiles = List(
+  def extFiles: List[ExtFile] = List(
     "css/bootstrap_dashboard.css",
     "css/bootstrap.min.css",
     "css/bootstrap-theme.min.css",
@@ -181,40 +187,63 @@ trait ReportBuilder extends ToolCommand {
       case _ =>
     }
 
+    _setRun = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
     _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
     _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
     _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
     _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)
 
-    logger.info("Copy Base files")
+    val baseFilesFuture = Future {
+      logger.info("Copy Base files")
 
-    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
-    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")
+      // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
+      val extOutputDir: File = new File(cmdArgs.outputDir, "ext")
 
-    // Copy each resource files out to the report destination
-    extFiles.par.foreach(
-      resource =>
-        IoUtils.copyStreamToFile(
-          getClass.getResourceAsStream(resource.resourcePath),
-          new File(extOutputDir, resource.targetPath),
-          createDirs = true)
-    )
+      // Copy each resource files out to the report destination
+      extFiles.foreach(
+        resource =>
+          IoUtils.copyStreamToFile(
+            getClass.getResourceAsStream(resource.resourcePath),
+            new File(extOutputDir, resource.targetPath),
+            createDirs = true)
+      )
+    }
 
-    total = ReportBuilder.countPages(indexPage)
-    logger.info(total + " pages to be generated")
+    val rootPage = indexPage.map { x => x.copy(subPages = x.subPages ::: generalPages(sampleId, libId)) }
 
+    //    total = ReportBuilder.countPages(rootPage)
     done = 0
 
     logger.info("Generate pages")
-    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
+    val jobsFutures = generatePage(summary, rootPage, cmdArgs.outputDir,
       args = pageArgs ++ cmdArgs.pageArgs.toMap ++
-        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
+        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
 
-    logger.info(jobs + " Done")
+    total = jobsFutures.size
+    logger.info(total + " pages to be generated")
+
+    def wait(futures: List[Future[Any]]): Unit = {
+      try {
+        Await.ready(Future.sequence(futures), Duration.fromNanos(30000000000L))
+      } catch {
+        case e: TimeoutException =>
+      }
+      val notDone = futures.filter(!_.isCompleted)
+      done += futures.size - notDone.size
+      if (notDone.nonEmpty) {
+        logger.info(s"$done / $total pages are generated")
+        wait(notDone)
+      }
+    }
+
+    wait(jobsFutures)
+    Await.ready(baseFilesFuture, Duration.Inf)
+
+    logger.info(s"Done, $done pages generated")
   }
 
   /** This must be implemented, this will be the root page of the report */
-  def indexPage: ReportPage
+  def indexPage: Future[ReportPage]
 
   /** This must be implemented, this will become the title of the report */
   def reportName: String
@@ -223,62 +252,113 @@ trait ReportBuilder extends ToolCommand {
    * This method will render the page and the subpages recursivly