Merge remote-tracking branch 'remotes/origin/develop' into fix-BIOPET-639

# Conflicts:
#	biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

Jenkinsfile

 node('local') {
-    timeout(45) {
-        try {
+    try {
 
-            stage('Init') {
-                tool 'JDK 8u102'
-                tool 'Apache Maven 3.3.9'
-            }
+        stage('Init') {
+            tool 'JDK 8u102'
+            tool 'Apache Maven 3.3.9'
+        }
 
+        timeout(45) {
             stage('Checkout') {
                 checkout scm
                 sh 'git submodule update --init --recursive'
             }
+        }
 
+        timeout(45) {
             stage('Build and Test') {
                 withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
                     sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
                 }
             }
+        }
 
-            stage('Report Tests') {
-                junit '*/target/surefire-reports/*.xml'
-            }
-
-            stage('Check git on changes') {
-                sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
-            }
+        stage('Report Tests') {
+            junit '*/target/surefire-reports/*.xml'
+        }
 
-            stage('Check Documentation') {
-                sh 'mkdocs build --clean --strict'
-            }
+        stage('Check git on changes') {
+            sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
+        }
 
-            if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
-                currentBuild.result = "SUCCESS"
-                slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
-            } else {
-                slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
-            }
+        stage('Check Documentation') {
+            sh 'mkdocs build --clean --strict'
+        }
 
-        } catch (e) {
-            if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
-                currentBuild.result = "FAILED"
-            }
-            slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+        if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
+            currentBuild.result = "SUCCESS"
+            slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+        } else {
+            slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+        }
 
-            throw e
+    } catch (e) {
+        if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
+            currentBuild.result = "FAILED"
         }
+        slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+
+        throw e
     }
 }

bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala

@@ -24,8 +24,7 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._
 
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 
 class BammetricsReport(val parent: Configurable) extends ReportBuilderExtension {
@@ -42,28 +41,27 @@ object BammetricsReport extends ReportBuilder {
   /** Name of report */
   val reportName = "Bam Metrics"
 
+  def pipelineName = "bammetrics"
+
   /** Root page for single BamMetrcis report */
-  def indexPage = {
-    val bamMetricsPage = this.bamMetricsPage(summary, sampleId, libId)
-    ReportPage(bamMetricsPage.subPages ::: List(
-      "Versions" -> ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
-      )), Map()),
-      "Files" -> ReportPage(List(), List(), Map())
-    ), List(
-      "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
-    ) ::: bamMetricsPage.sections,
-      Map()
-    )
-  }
+  def indexPage: Future[ReportPage] =
+    bamMetricsPage(summary, sampleId, libId).map { bamMetricsPage =>
+      ReportPage(bamMetricsPage.subPages ::: List(
+        "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
+        )), Map())),
+        "Files" -> filesPage(sampleId, libId)
+      ), List(
+        "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
+      ) ::: bamMetricsPage.sections,
+        Map()
+      )
+    }
 
   /** Generates a page with alignment stats */
   def bamMetricsPage(summary: SummaryDb,
                      sampleId: Option[Int],
                      libId: Option[Int],
-                     metricsTag: String = "bammetrics") = {
-
-    //val pipelineId: Int = summary.getPipelineId(runId, metricsTag).map(_.get)
-
+                     metricsTag: String = "bammetrics"): Future[ReportPage] = Future {
     val wgsExecuted = summary.getStatsSize(runId, metricsTag, "wgs", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
     val rnaExecuted = summary.getStatsSize(runId, metricsTag, "rna", sample = sampleId.map(SampleId), library = libId.map(LibraryId)) >= 1
 
@@ -85,10 +83,10 @@ object BammetricsReport extends ReportBuilder {
 
     ReportPage(
       if (targets.isEmpty) List()
-      else List("Targets" -> ReportPage(
+      else List("Targets" -> Future.successful(ReportPage(
         List(),
         targets.map(t => t -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp", Map("target" -> Some(t)))),
-        Map())),
+        Map()))),
       List(
         "Summary" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"),
         "Mapping Quality" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/mappingQuality.ssp", Map("showPlot" -> true)),
@@ -161,7 +159,7 @@ object BammetricsReport extends ReportBuilder {
     plot.output = pngFile
     plot.ylabel = Some("Reads")
     plot.width = Some(200 + (results.size * 10))
-    plot.title = Some("Aligned reads")
+    plot.title = Some("Aligned_reads")
     plot.runLocal()
   }
 

biopet-core/src/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp

 #import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
 #import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
+#import(scala.concurrent.Future)
 <%@ var summary: SummaryDb %>
-<%@ var indexPage: ReportPage %>
+<%@ var indexPage: Future[ReportPage] %>
 <%@ var reportName: String %>
 <%@ var page: ReportPage %>
 <%@ var path: List[String] %>
 <%@ var args: Map[String, Any] %>
 <%@ var rootPath: String %>
 #{
+    val iPage = Await.result(indexPage, Duration.Inf)
+
     def createMenu(page: ReportPage, path: List[String] = Nil, first: Boolean = true): String = {
         val buffer: StringBuffer = new StringBuffer()
 
@@ -15,11 +20,13 @@
             buffer.append("<ul class=\"dropdown-menu list-group\">")
         }
 
-        for (subPage <- page.subPages.sortBy(_._1)) {
+        for (subPageFuture <- page.subPages.sortBy(_._1)) {
 
+            val subPageName = subPageFuture._1
+            val subPage = Await.result(subPageFuture._2, Duration.Inf)
             val href: String = {
-                if (path.isEmpty) rootPath + subPage._1 + "/index.html"
-                else rootPath + path.mkString("","/","/") + subPage._1 + "/index.html"
+                if (path.isEmpty) rootPath + subPageName + "/index.html"
+                else rootPath + path.mkString("","/","/") + subPageName + "/index.html"
             }
 
 //            buffer.append("<li")
@@ -30,16 +37,16 @@
 //            buffer.append(createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false))
 //            buffer.append("</li>")
 
-            val listSubmenu = if(subPage._2.subPages.nonEmpty) "dropdown-submenu" else ""
+            val listSubmenu = if(subPage.subPages.nonEmpty) "dropdown-submenu" else ""
 //            val subMenuBadgeCount = if(subPage._2.subPages.nonEmpty && first) "<span class='badge'>%d</span>".format(subPage._2.subPages.size) else ""
             val tabIndex = if (first) " tabindex='-1'" else ""
 //            val listGroupA = if(subPage._2.subPages.nonEmpty) "list-group-item" else ""
 
             var menuItem: String = "<li class='%s'>".format(listSubmenu) +
             "<a href='%s' class='%s'%s>".format(href, "", tabIndex) +
-            "%s".format(subPage._1) +
+            "%s".format(subPageName) +
             "</a>" +
-            createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false) +
+            createMenu(subPage, path ::: subPageName :: Nil, first = false) +
             "</li>"
             buffer.append(menuItem)
 
@@ -51,10 +58,10 @@
     }
 
     def getSubPage(path:List[String]): ReportPage = {
-        path.foldLeft(indexPage)((c, p) => {
+        path.foldLeft(iPage)((c, p) => {
             val foundPages = c.subPages.filter(_._1 == p)
             require(foundPages.size == 1, "Name of page not found or duplicate is found, page:: " + p + " in path: " + path)
-            foundPages.head._2
+            Await.result(foundPages.head._2, Duration.Inf)
         })
     }
 }#
@@ -111,9 +118,9 @@
         <ul class="nav navbar-nav">
             <li class="root #if (path.isEmpty) active #end">
                 <a class="navbar-brand" href="${rootPath}index.html">${reportName}
-                    #if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
+                    #if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
                 </a>
-                ${unescape(createMenu(indexPage))}
+                ${unescape(createMenu(iPage))}
             </li>
         </ul>
     </div>
@@ -126,9 +133,9 @@
                 #if(t == 0)
                     <li class="root">
                         <a href="${rootPath}index.html">Home
-                            #if (indexPage.subPages.nonEmpty) <b class="caret"></b> #end
+                            #if (iPage.subPages.nonEmpty) <b class="caret"></b> #end
                         </a>
-                        ${unescape(createMenu(indexPage, Nil, false))}
+                        ${unescape(createMenu(iPage, Nil, false))}
                     </li>
                 #else
                     <li class="root #if (t == path.size) active #end">

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala

@@ -14,7 +14,7 @@
  */
 package nl.lumc.sasc.biopet.core.report
 
-import scala.concurrent.Await
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration.Duration
 
 /**
@@ -25,46 +25,44 @@ import scala.concurrent.duration.Duration
 trait MultisampleReportBuilder extends ReportBuilder {
 
   /** Method to generate a single sample page */
-  def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage
+  def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of samples, can be override */
   def samplesSections: List[(String, ReportSection)] = {
-    List(
-      ("Samples", ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
-    )
+    List("Samples" -> ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
   }
 
   /** Method to generate a single library page */
-  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): ReportPage
+  def libraryPage(sampleId: Int, libraryId: Int, args: Map[String, Any]): Future[ReportPage]
 
   /** Default list of libraries, can be override */
   def librariesSections: List[(String, ReportSection)] = {
-    List(
-      ("Libraries", ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
-    )
+    List("Libraries" -> ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
   }
 
   /** Generate the samples page including a single sample page for each sample in the summary */
-  def generateSamplesPage(args: Map[String, Any]): ReportPage = {
-    val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
-    val samplePages = samples.map(_.id)
-      .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId))))
-      .toList
-    ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
-  }
+  def generateSamplesPage(args: Map[String, Any]): Future[ReportPage] =
+    summary.getSamples(runId = Some(runId)).map { samples =>
+      val samplePages = samples.map(_.id)
+        .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId)))
+          .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, None) :: Nil)))
+        .toList
+      ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
+    }
 
   /** Generate the libraries page for a single sample with a subpage for eacht library */
-  def generateLibraryPage(args: Map[String, Any]): ReportPage = {
+  def generateLibraryPage(args: Map[String, Any]): Future[ReportPage] = {
     val sampleId = args("sampleId") match {
       case Some(x: Int) => x
       case None         => throw new IllegalStateException("Sample not found")
     }
 
-    val libraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)), Duration.Inf)
-
-    val libPages = libraries.map(_.id)
-      .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId))))
-      .toList
-    ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
+    summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)).map { libraries =>
+      val libPages = libraries.map(_.id)
+        .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId)))
+          .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, libId) :: Nil)))
+        .toList
+      ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
+    }
   }
 }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -19,12 +19,13 @@ import java.io._
 import nl.lumc.sasc.biopet.core.ToolCommandFunction
 import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample, Run }
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ LibraryId, SampleId }
 import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
 import org.broadinstitute.gatk.utils.commandline.Input
 import org.fusesource.scalate.TemplateEngine
 
 import scala.collection.mutable
-import scala.concurrent.{ Await, Future }
+import scala.concurrent._
 import scala.concurrent.duration.Duration
 import scala.language.postfixOps
 import scala.language.implicitConversions
@@ -73,6 +74,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
 
 trait ReportBuilder extends ToolCommand {
 
+  implicit lazy val ec = ReportBuilder.ec
   implicit def toOption[T](x: T): Option[T] = Option(x)
   implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
 
@@ -191,35 +193,57 @@ trait ReportBuilder extends ToolCommand {
     _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
     _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)
 
-    logger.info("Copy Base files")
+    val baseFilesFuture = Future {
+      logger.info("Copy Base files")
 
-    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
-    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")
+      // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
+      val extOutputDir: File = new File(cmdArgs.outputDir, "ext")
 
-    // Copy each resource files out to the report destination
-    extFiles.par.foreach(
-      resource =>
-        IoUtils.copyStreamToFile(
-          getClass.getResourceAsStream(resource.resourcePath),
-          new File(extOutputDir, resource.targetPath),
-          createDirs = true)
-    )
+      // Copy each resource files out to the report destination
+      extFiles.foreach(
+        resource =>
+          IoUtils.copyStreamToFile(
+            getClass.getResourceAsStream(resource.resourcePath),
+            new File(extOutputDir, resource.targetPath),
+            createDirs = true)
+      )
+    }
 
-    total = ReportBuilder.countPages(indexPage)
-    logger.info(total + " pages to be generated")
+    val rootPage = indexPage.map { x => x.copy(subPages = x.subPages ::: generalPages(sampleId, libId)) }
 
+    //    total = ReportBuilder.countPages(rootPage)
     done = 0
 
     logger.info("Generate pages")
-    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
+    val jobsFutures = generatePage(summary, rootPage, cmdArgs.outputDir,
       args = pageArgs ++ cmdArgs.pageArgs.toMap ++
-        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
+        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
+
+    total = jobsFutures.size
+    logger.info(total + " pages to be generated")
 
-    logger.info(jobs + " Done")
+    def wait(futures: List[Future[Any]]): Unit = {
+      try {
+        Await.ready(Future.sequence(futures), Duration.fromNanos(30000000000L))
+      } catch {
+        case e: TimeoutException =>
+      }
+      val notDone = futures.filter(!_.isCompleted)
+      done += futures.size - notDone.size
+      if (notDone.nonEmpty) {
+        logger.info(s"$done / $total pages are generated")
+        wait(notDone)
+      }
+    }
+
+    wait(jobsFutures)
+    Await.ready(baseFilesFuture, Duration.Inf)
+
+    logger.info(s"Done, $done pages generated")
   }
 
   /** This must be implemented, this will be the root page of the report */
-  def indexPage: ReportPage
+  def indexPage: Future[ReportPage]
 
   /** This must be implemented, this will become the title of the report */
   def reportName: String
@@ -228,63 +252,113 @@ trait ReportBuilder extends ToolCommand {
    * This method will render the page and the subpages recursivly
    *
    * @param summary The summary object
-   * @param page Page to render
+   * @param pageFuture Page to render
    * @param outputDir Root output dir of the report
    * @param path Path from root to current page
    * @param args Args to add to this sub page, are args from current page are passed automaticly
    * @return Number of pages including all subpages that are rendered
    */
   def generatePage(summary: SummaryDb,
-                   page: ReportPage,
+                   pageFuture: Future[ReportPage],
                    outputDir: File,
                    path: List[String] = Nil,
-                   args: Map[String, Any] = Map()): Int = {
-
+                   args: Map[String, Any] = Map()): List[Future[_]] = {
     val pageOutputDir = new File(outputDir, path.mkString(File.separator))
-    pageOutputDir.mkdirs()
-    val rootPath = "./" + Array.fill(path.size)("../").mkString
-    val pageArgs = args ++ page.args ++
-      Map("page" -> page,
-        "run" -> run,
-        "path" -> path,
-        "outputDir" -> pageOutputDir,
-        "rootPath" -> rootPath,
-        "allPipelines" -> pipelines,
-        "allModules" -> modules,
-        "allSamples" -> samples,
-        "allLibraries" -> libraries
-      )
 
-    // Generating subpages
-    val jobs = page.subPages.par.flatMap {
-      case (name, subPage) => Some(generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs))
-      case _               => None
+    def pageArgs(page: ReportPage) = {
+      val rootPath = "./" + Array.fill(path.size)("../").mkString
+      args ++ page.args ++
+        Map("page" -> page,
+          "run" -> run,
+          "path" -> path,
+          "outputDir" -> pageOutputDir,
+          "rootPath" -> rootPath,
+          "allPipelines" -> pipelines,
+          "allModules" -> modules,
+          "allSamples" -> samples,
+          "allLibraries" -> libraries
+        )
     }
 
-    val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
-      pageArgs ++ Map("args" -> pageArgs))
+    val subPageJobs = pageFuture.map { page =>
+      // Generating subpages
+      page.subPages.flatMap {
+        case (name, subPage) => generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs(page))
+      }
+    }
 
-    val file = new File(pageOutputDir, "index.html")
-    val writer = new PrintWriter(file)
-    writer.println(output)
-    writer.close()
+    val renderFuture = pageFuture.map { page =>
+      pageOutputDir.mkdirs()
 
-    done += 1
-    if (done % 100 == 0) logger.info(done + " Done, " + (done.toDouble / total * 100) + "%")
-    jobs.sum + 1
+      val file = new File(pageOutputDir, "index.html")
+      logger.info(s"Start rendering: $file")
+
+      val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
+        pageArgs(page) ++ Map("args" -> pageArgs(page)))
+
+      val writer = new PrintWriter(file)
+      writer.println(output)
+      writer.close()
+      logger.info(s"Done rendering: $file")
+
+    }
+
+    renderFuture :: Await.result(subPageJobs, Duration.Inf)
   }
+
+  def pipelineName: String
+
+  /** Files page, can be used general or at sample level */
+  def filesPage(sampleId: Option[Int] = None, libraryId: Option[Int] = None): Future[ReportPage] = {
+    val dbFiles = summary.getFiles(runId, sample = sampleId.map(SampleId),
+      library = libraryId.map(LibraryId))
+      .map(_.groupBy(_.pipelineId))
+    val modulePages = dbFiles.map(_.map {
+      case (pipelineId, files) =>
+        val moduleSections = files.groupBy(_.moduleId).map {
+          case (moduleId, files) =>
+            val moduleName: Future[String] = moduleId match {
+              case Some(id) => summary.getModuleName(pipelineId, id).map(_.getOrElse("Pipeline"))
+              case _        => Future.successful("Pipeline")
+            }
+            moduleName.map(_ -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> files)))
+        }
+        val moduleSectionsSorted = moduleSections.find(_._1 == "Pipeline") ++ moduleSections.filter(_._1 != "Pipeline")
+        summary.getPipelineName(pipelineId = pipelineId)
+          .map(_.get -> Future.sequence(moduleSectionsSorted)
+            .map(sections => ReportPage(Nil, sections.toList, Map())))
+    })
+
+    val pipelineFiles = summary.getPipelineId(runId, pipelineName)
+      .flatMap(pipelinelineId => dbFiles
+        .map(x => x.get(pipelinelineId.get).getOrElse(Seq())
+          .filter(_.moduleId.isEmpty)))
+
+    modulePages.flatMap(Future.sequence(_)).map(x => ReportPage(x.toList,
+      s"$pipelineName files" -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> Await.result(pipelineFiles, Duration.Inf))) ::
+        "Sub pipelines/modules" -> ReportSection("/nl/lumc/sasc/biopet/core/report/fileModules.ssp", Map("pipelineIds" -> Await.result(dbFiles.map(_.keys.toList), Duration.Inf))) :: Nil, Map()))
+  }
+
+  /** This generate general pages that all reports should have */
+  def generalPages(sampleId: Option[Int], libId: Option[Int]): List[(String, Future[ReportPage])] = List(
+    "Versions" -> Future.successful(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp")), Map())),
+    "Files" -> filesPage(sampleId, libId)
+  )
+
 }
 
 object ReportBuilder {
 
+  implicit lazy val ec = ExecutionContext.global
+
   /** Single template render engine, this will have a cache for all compile templates */
   protected val engine = new TemplateEngine()
   engine.allowReload = false
 
   /** This will give the total number of pages including all nested pages */
-  def countPages(page: ReportPage): Int = {
-    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
-  }
+  //  def countPages(page: ReportPage): Int = {
+  //    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
+  //  }