Switch core to futures

biopet-core/src/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp

 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
 #import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
 <%@ var summary: SummaryDb %>
 <%@ var indexPage: ReportPage %>
 <%@ var reportName: String %>
@@ -15,11 +17,13 @@
             buffer.append("<ul class=\"dropdown-menu list-group\">")
         }
 
-        for (subPage <- page.subPages.sortBy(_._1)) {
+        for (subPageFuture <- page.subPages.sortBy(_._1)) {
 
+            val subPageName = subPageFuture._1
+            val subPage = Await.result(subPageFuture._2, Duration.Inf)
             val href: String = {
-                if (path.isEmpty) rootPath + subPage._1 + "/index.html"
-                else rootPath + path.mkString("","/","/") + subPage._1 + "/index.html"
+                if (path.isEmpty) rootPath + subPageName + "/index.html"
+                else rootPath + path.mkString("","/","/") + subPageName + "/index.html"
             }
 
 //            buffer.append("<li")
@@ -30,16 +34,16 @@
 //            buffer.append(createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false))
 //            buffer.append("</li>")
 
-            val listSubmenu = if(subPage._2.subPages.nonEmpty) "dropdown-submenu" else ""
+            val listSubmenu = if(subPage.subPages.nonEmpty) "dropdown-submenu" else ""
 //            val subMenuBadgeCount = if(subPage._2.subPages.nonEmpty && first) "<span class='badge'>%d</span>".format(subPage._2.subPages.size) else ""
             val tabIndex = if (first) " tabindex='-1'" else ""
 //            val listGroupA = if(subPage._2.subPages.nonEmpty) "list-group-item" else ""
 
             var menuItem: String = "<li class='%s'>".format(listSubmenu) +
             "<a href='%s' class='%s'%s>".format(href, "", tabIndex) +
-            "%s".format(subPage._1) +
+            "%s".format(subPageName) +
             "</a>" +
-            createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false) +
+            createMenu(subPage, path ::: subPageName :: Nil, first = false) +
             "</li>"
             buffer.append(menuItem)
 
@@ -54,7 +58,7 @@
         path.foldLeft(indexPage)((c, p) => {
             val foundPages = c.subPages.filter(_._1 == p)
             require(foundPages.size == 1, "Name of page not found or duplicate is found, page:: " + p + " in path: " + path)
-            foundPages.head._2
+            Await.result(foundPages.head._2, Duration.Inf)
         })
     }
 }#

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala

@@ -14,8 +14,9 @@
  */
 package nl.lumc.sasc.biopet.core.report
 
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
+import scala.concurrent.ExecutionContext.Implicits.global
 
 /**
  * This trait will generate a report with added function to generate sample and library pages for those existing in the summary.
@@ -25,7 +26,7 @@ import scala.concurrent.duration.Duration
 trait MultisampleReportBuilder extends ReportBuilder {
 
   /** Method to generate a single sample page */
-  def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage
+  def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of samples, can be override */
   def samplesSections: List[(String, ReportSection)] = {
@@ -35,7 +36,7 @@ trait MultisampleReportBuilder extends ReportBuilder {
   }
 
   /** Method to generate a single library page */
-  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): ReportPage
+  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of libraries, can be override */
   def librariesSections: List[(String, ReportSection)] = {
@@ -45,7 +46,7 @@ trait MultisampleReportBuilder extends ReportBuilder {
   }
 
   /** Generate the samples page including a single sample page for each sample in the summary */
-  def generateSamplesPage(args: Map[String, Any]): ReportPage = {
+  def generateSamplesPage(args: Map[String, Any]): Future[ReportPage] = Future {
     val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
     val samplePages = samples.map(_.id)
       .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId))))
@@ -54,7 +55,7 @@ trait MultisampleReportBuilder extends ReportBuilder {
   }
 
   /** Generate the libraries page for a single sample with a subpage for eacht library */
-  def generateLibraryPage(args: Map[String, Any]): ReportPage = {
+  def generateLibraryPage(args: Map[String, Any]): Future[ReportPage] = Future {
     val sampleId = args("sampleId") match {
       case Some(x: Int) => x
       case None         => throw new IllegalStateException("Sample not found")

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -28,6 +28,7 @@ import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 import scala.language.postfixOps
 import scala.language.implicitConversions
+import scala.concurrent.ExecutionContext.Implicits.global
 
 /**
  * This trait is meant to make an extension for a report object
@@ -74,7 +75,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
 trait ReportBuilder extends ToolCommand {
 
   implicit def toOption[T](x: T): Option[T] = Option(x)
-  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
+  //  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
 
   case class Args(summaryDbFile: File = null,
                   outputDir: File = null,
@@ -186,6 +187,7 @@ trait ReportBuilder extends ToolCommand {
     _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
     _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)
 
+    // TODO: switch to future for base files
     logger.info("Copy Base files")
 
     // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
@@ -200,21 +202,23 @@ trait ReportBuilder extends ToolCommand {
           createDirs = true)
     )
 
-    total = ReportBuilder.countPages(indexPage)
+    val rootPage = indexPage
+
+    //    total = ReportBuilder.countPages(rootPage)
     logger.info(total + " pages to be generated")
 
     done = 0
 
     logger.info("Generate pages")
-    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
+    val jobs = generatePage(summary, rootPage, cmdArgs.outputDir,
       args = pageArgs ++ cmdArgs.pageArgs.toMap ++
-        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
+        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
 
-    logger.info(jobs + " Done")
+    Await.result(jobs, Duration.Inf)
   }
 
   /** This must be implemented, this will be the root page of the report */
-  def indexPage: ReportPage
+  def indexPage: Future[ReportPage]
 
   /** This must be implemented, this will become the title of the report */
   def reportName: String
@@ -223,49 +227,50 @@ trait ReportBuilder extends ToolCommand {
    * This method will render the page and the subpages recursivly
    *
    * @param summary The summary object
-   * @param page Page to render
+   * @param pageFuture Page to render
    * @param outputDir Root output dir of the report
    * @param path Path from root to current page
    * @param args Args to add to this sub page, are args from current page are passed automaticly
    * @return Number of pages including all subpages that are rendered
    */
   def generatePage(summary: SummaryDb,
-                   page: ReportPage,
+                   pageFuture: Future[ReportPage],
                    outputDir: File,
                    path: List[String] = Nil,
-                   args: Map[String, Any] = Map()): Int = {
-
-    val pageOutputDir = new File(outputDir, path.mkString(File.separator))
-    pageOutputDir.mkdirs()
-    val rootPath = "./" + Array.fill(path.size)("../").mkString
-    val pageArgs = args ++ page.args ++
-      Map("page" -> page,
-        "path" -> path,
-        "outputDir" -> pageOutputDir,
-        "rootPath" -> rootPath,
-        "allPipelines" -> pipelines,
-        "allModules" -> modules,
-        "allSamples" -> samples,
-        "allLibraries" -> libraries
-      )
-
-    // Generating subpages
-    val jobs = page.subPages.par.flatMap {
-      case (name, subPage) => Some(generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs))
-      case _               => None
+                   args: Map[String, Any] = Map()): Future[_] = {
+    pageFuture.flatMap { page =>
+      val pageOutputDir = new File(outputDir, path.mkString(File.separator))
+      pageOutputDir.mkdirs()
+      val rootPath = "./" + Array.fill(path.size)("../").mkString
+      val pageArgs = args ++ page.args ++
+        Map("page" -> page,
+          "path" -> path,
+          "outputDir" -> pageOutputDir,
+          "rootPath" -> rootPath,
+          "allPipelines" -> pipelines,
+          "allModules" -> modules,
+          "allSamples" -> samples,
+          "allLibraries" -> libraries
+        )
+
+      // Generating subpages
+      val jobs = Future.sequence(page.subPages.map {
+        case (name, subPage) => generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs)
+      })
+
+      val renderFuture = Future {
+        val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
+          pageArgs ++ Map("args" -> pageArgs))
+
+        val file = new File(pageOutputDir, "index.html")
+        val writer = new PrintWriter(file)
+        writer.println(output)
+        writer.close()
+      }
+
+      Future.sequence(renderFuture :: jobs :: Nil)
     }
 
-    val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
-      pageArgs ++ Map("args" -> pageArgs))
-
-    val file = new File(pageOutputDir, "index.html")
-    val writer = new PrintWriter(file)
-    writer.println(output)
-    writer.close()
-
-    done += 1
-    if (done % 100 == 0) logger.info(done + " Done, " + (done.toDouble / total * 100) + "%")
-    jobs.sum + 1
   }
 }
 
@@ -276,9 +281,9 @@ object ReportBuilder {
   engine.allowReload = false
 
   /** This will give the total number of pages including all nested pages */
-  def countPages(page: ReportPage): Int = {
-    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
-  }
+  //  def countPages(page: ReportPage): Int = {
+  //    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
+  //  }
 
   /**
    * This method will render a template that is located in the classpath / jar

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportPage.scala

@@ -14,6 +14,8 @@
  */
 package nl.lumc.sasc.biopet.core.report
 
+import scala.concurrent.Future
+
 /**
  * Created by pjvan_thof on 3/27/15.
  *
@@ -21,6 +23,6 @@ package nl.lumc.sasc.biopet.core.report
  * @param sections Sections for this page
  * @param args Arguments for this page, this arguments get passed to all section and subpages
  */
-case class ReportPage(subPages: List[(String, ReportPage)],
+case class ReportPage(subPages: List[(String, Future[ReportPage])],
                       sections: List[(String, ReportSection)],
                       args: Map[String, Any])

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilderTest.scala

@@ -23,8 +23,9 @@ import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.Test
 
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
+import scala.concurrent.ExecutionContext.Implicits.global
 
 /**
  * Created by pjvanthof on 24/02/16.
@@ -38,12 +39,12 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
   def testGeneratePages(): Unit = {
     val builder = new MultisampleReportBuilder {
       def reportName: String = "test"
-      def indexPage: ReportPage = ReportPage("Samples" -> generateSamplesPage(Map()) :: Nil, Nil, Map())
+      def indexPage: Future[ReportPage] = Future(ReportPage("Samples" -> generateSamplesPage(Map()) :: Nil, Nil, Map()))
 
-      def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage =
-        ReportPage("Libraries" -> generateLibraryPage(Map("sampleId" -> Some(sampleId))) :: Nil, Nil, Map())
+      def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage] =
+        Future(ReportPage("Libraries" -> generateLibraryPage(Map("sampleId" -> Some(sampleId))) :: Nil, Nil, Map()))
 
-      def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = ReportPage(Nil, Nil, Map())
+      def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]) = Future(ReportPage(Nil, Nil, Map()))
     }
     val dbFile = File.createTempFile("summary.", ".db")
     dbFile.deleteOnExit()

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala

@@ -23,9 +23,11 @@ import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.{ DataProvider, Test }
 
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 
+import scala.concurrent.ExecutionContext.Implicits.global
+
 /**
  * Created by pjvanthof on 24/02/16.
  */
@@ -47,8 +49,8 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
   def testGeneratePages(sample: Option[String], lib: Option[String], nested: Boolean): Unit = {
     val builder = new ReportBuilder {
       def reportName: String = "test"
-      def indexPage: ReportPage = ReportPage(
-        (if (nested) "p1" -> ReportPage(Nil, Nil, Map()) :: Nil else Nil), Nil, Map())
+      def indexPage: Future[ReportPage] = Future(ReportPage(
+        (if (nested) "p1" -> Future(ReportPage(Nil, Nil, Map())) :: Nil else Nil), Nil, Map()))
     }
 
     val dbFile = File.createTempFile("summary.", ".db")
@@ -77,22 +79,22 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
     db.close()
   }
 
-  @Test
-  def testCountPages: Unit = {
-    ReportBuilder.countPages(ReportPage(Nil, Nil, Map())) shouldBe 1
-    ReportBuilder.countPages(ReportPage(
-      "p1" -> ReportPage(Nil, Nil, Map()) :: Nil,
-      Nil, Map())) shouldBe 2
-    ReportBuilder.countPages(ReportPage(
-      "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage(Nil, Nil, Map()) :: Nil,
-      Nil, Map())) shouldBe 3
-    ReportBuilder.countPages(ReportPage(
-      "p1" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
-      Nil, Map())) shouldBe 3
-    ReportBuilder.countPages(ReportPage(
-      "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
-      Nil, Map())) shouldBe 4
-  }
+  //  @Test
+  //  def testCountPages: Unit = {
+  //    ReportBuilder.countPages(ReportPage(Nil, Nil, Map())) shouldBe 1
+  //    ReportBuilder.countPages(ReportPage(
+  //      "p1" -> ReportPage(Nil, Nil, Map()) :: Nil,
+  //      Nil, Map())) shouldBe 2
+  //    ReportBuilder.countPages(ReportPage(
+  //      "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage(Nil, Nil, Map()) :: Nil,
+  //      Nil, Map())) shouldBe 3
+  //    ReportBuilder.countPages(ReportPage(
+  //      "p1" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
+  //      Nil, Map())) shouldBe 3
+  //    ReportBuilder.countPages(ReportPage(
+  //      "p1" -> ReportPage(Nil, Nil, Map()) :: "p2" -> ReportPage("p1" -> ReportPage(Nil, Nil, Map()) :: Nil, Nil, Map()) :: Nil,
+  //      Nil, Map())) shouldBe 4
+  //  }
 
   @Test
   def testRenderTemplate: Unit = {