generelize files and version even more

bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala

@@ -41,13 +41,15 @@ object BammetricsReport extends ReportBuilder {
   /** Name of report */
   val reportName = "Bam Metrics"
 
+  def pipelineName = "bammetrics"
+
   /** Root page for single BamMetrcis report */
   def indexPage: Future[ReportPage] = Future {
     val bamMetricsPage = Await.result(this.bamMetricsPage(summary, sampleId, libId), Duration.Inf)
     ReportPage(bamMetricsPage.subPages ::: List(
       "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
       )), Map())),
-      "Files" -> Future(ReportPage(List(), List(), Map()))
+      "Files" -> filesPage(sampleId, libId)
     ), List(
       "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp")
     ) ::: bamMetricsPage.sections,

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala

@@ -29,9 +29,7 @@ trait MultisampleReportBuilder extends ReportBuilder {
 
   /** Default list of samples, can be override */
   def samplesSections: List[(String, ReportSection)] = {
-    List(
-      ("Samples", ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
-    )
+    List("Samples" -> ReportSection("/nl/lumc/sasc/biopet/core/report/samplesList.ssp"))
   }
 
   /** Method to generate a single library page */
@@ -39,16 +37,15 @@ trait MultisampleReportBuilder extends ReportBuilder {
 
   /** Default list of libraries, can be override */
   def librariesSections: List[(String, ReportSection)] = {
-    List(
-      ("Libraries", ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
-    )
+    List("Libraries" -> ReportSection("/nl/lumc/sasc/biopet/core/report/librariesList.ssp"))
   }
 
   /** Generate the samples page including a single sample page for each sample in the summary */
   def generateSamplesPage(args: Map[String, Any]): Future[ReportPage] = Future {
     val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
     val samplePages = samples.map(_.id)
-      .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId))))
+      .map(sampleId => sampleId -> samplePage(sampleId, args ++ Map("sampleId" -> Some(sampleId)))
+        .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, None) :: Nil)))
       .toList
     ReportPage(samplePages.map(x => samples.find(_.id == x._1).get.name -> x._2), samplesSections, args)
   }
@@ -63,7 +60,8 @@ trait MultisampleReportBuilder extends ReportBuilder {
     val libraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = Some(sampleId)), Duration.Inf)
 
     val libPages = libraries.map(_.id)
-      .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId))))
+      .map(libId => libId -> libraryPage(sampleId, libId, args ++ Map("libId" -> Some(libId)))
+        .map(x => x.copy(subPages = x.subPages ::: "Files" -> filesPage(sampleId, libId) :: Nil)))
       .toList
     ReportPage(libPages.map(x => libraries.find(_.id == x._1).get.name -> x._2), librariesSections, args)
   }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -17,9 +17,10 @@ package nl.lumc.sasc.biopet.core.report
 import java.io._
 
 import nl.lumc.sasc.biopet.core.ToolCommandFunction
-import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
+import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Sample}
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
-import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
+import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{LibraryId, SampleId}
+import nl.lumc.sasc.biopet.utils.{IoUtils, Logging, ToolCommand}
 import org.broadinstitute.gatk.utils.commandline.Input
 import org.fusesource.scalate.TemplateEngine
 
@@ -203,7 +204,7 @@ trait ReportBuilder extends ToolCommand {
       )
     }
 
-    val rootPage = indexPage
+    val rootPage = indexPage.map { x => x.copy(subPages = x.subPages ::: generalPages(sampleId, libId))}
 
     //    total = ReportBuilder.countPages(rootPage)
     logger.info(total + " pages to be generated")
@@ -274,6 +275,41 @@ trait ReportBuilder extends ToolCommand {
     }
 
   }
+
+  def pipelineName
+
+  /** Files page, can be used general or at sample level */
+  def filesPage(sampleId: Option[Int] = None, libraryId: Option[Int] = None): Future[ReportPage] = {
+    val dbFiles = summary.getFiles(runId, sample = sampleId.map(SampleId),
+      library = libraryId.map(LibraryId))
+      .map(_.groupBy(_.pipelineId))
+    val modulePages = dbFiles.map(_.map {
+      case (pipelineId, files) =>
+        val moduleSections = files.groupBy(_.moduleId).map {
+          case (moduleId, files) =>
+            val moduleName: Future[String] = moduleId match {
+              case Some(id) => summary.getModuleName(pipelineId, id).map(_.getOrElse("Pipeline"))
+              case _        => Future("Pipeline")
+            }
+            moduleName.map(_ -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> files)))
+        }
+        val moduleSectionsSorted = moduleSections.find(_._1 == "Pipeline") ++ moduleSections.filter(_._1 != "Pipeline")
+        summary.getPipelineName(pipelineId = pipelineId).map(_.get -> Future(ReportPage(Nil, Await.result(Future.sequence(moduleSectionsSorted), Duration.Inf).toList, Map())))
+    })
+
+    val pipelineFiles = summary.getPipelineId(runId, pipelineName).flatMap(pipelinelineId => dbFiles.map(x => x(pipelinelineId.get).filter(_.moduleId.isEmpty)))
+
+    modulePages.flatMap(Future.sequence(_)).map(x => ReportPage(x.toList,
+      s"$pipelineName files" -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> Await.result(pipelineFiles, Duration.Inf))) ::
+        "Sub pipelines/modules" -> ReportSection("/nl/lumc/sasc/biopet/core/report/fileModules.ssp", Map("pipelineIds" -> Await.result(dbFiles.map(_.keys.toList), Duration.Inf))) :: Nil, Map()))
+  }
+
+  /** This generate general pages that all reports should have */
+  def generalPages(sampleId: Option[Int], libId: Option[Int]): List[(String, Future[ReportPage])] = List(
+    "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp")), Map())),
+    "Files" -> filesPage(sampleId, libId)
+  )
+
 }
 
 object ReportBuilder {

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilderTest.scala

@@ -37,6 +37,7 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
   @Test
   def testGeneratePages(): Unit = {
     val builder = new MultisampleReportBuilder {
+      def pipelineName = "test"
       def reportName: String = "test"
       def indexPage: Future[ReportPage] = Future(ReportPage("Samples" -> generateSamplesPage(Map()) :: Nil, Nil, Map()))
 

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala

@@ -46,6 +46,7 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
   @Test(dataProvider = "testGeneratePages")
   def testGeneratePages(sample: Option[String], lib: Option[String], nested: Boolean): Unit = {
     val builder = new ReportBuilder {
+      def pipelineName = "test"
       def reportName: String = "test"
       def indexPage: Future[ReportPage] = Future(ReportPage(
         (if (nested) "p1" -> Future(ReportPage(Nil, Nil, Map())) :: Nil else Nil), Nil, Map()))

flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/FlexiprepReport.scala

@@ -37,18 +37,14 @@ class FlexiprepReport(val parent: Configurable) extends ReportBuilderExtension {
 object FlexiprepReport extends ReportBuilder {
   val reportName = "Flexiprep"
 
+  def pipelineName = "flexiprep"
+
   override def pageArgs = Map("multisample" -> false)
 
   /** Index page for a flexiprep report */
   def indexPage: Future[ReportPage] = Future {
     val flexiprepPage = this.flexiprepPage
-    ReportPage(List("Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
-    )), Map())),
-      "Files" -> Future(ReportPage(List(), List( //TODO: Fix files
-      //        "Input fastq files" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepInputfiles.ssp"),
-      //        "After QC fastq files" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepOutputfiles.ssp")
-      ), Map()))
-    ), List(
+    ReportPage(Nil, List(
       "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepFront.ssp")
     ) ::: Await.result(flexiprepPage, Duration.Inf).sections,
       Map()

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsReport.scala

@@ -36,6 +36,8 @@ class GearsReport(val parent: Configurable) extends ReportBuilderExtension {
 
 object GearsReport extends MultisampleReportBuilder {
 
+  def pipelineName = "gears"
+
   def reportName = "Gears Report"
 
   override def extFiles = super.extFiles ++ List("js/gears.js", "js/krona-2.0.js", "img/krona/loading.gif", "img/krona/hidden.png", "img/krona/favicon.ico")
@@ -49,28 +51,33 @@ object GearsReport extends MultisampleReportBuilder {
     val qiimeClosesOtuTable = summary.getFile(runId, "gears", key = "qiime_closed_otu_table")
     val qiimeOpenOtuTable = summary.getFile(runId, "gears", key = "qiime_open_otu_table")
 
+    val centrifugePage = (if (centrifugeExecuted) Some("Centriguge analysis" -> Future(ReportPage(List("Non-unique" ->
+      Future(ReportPage(List(), List("All mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
+        Map("summaryStatsTag" -> "centrifuge_report")
+      )), Map()))), List(
+      "Unique mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
+        Map("summaryStatsTag" -> "centrifuge_unique_report")
+      )), Map("summaryModuleTag" -> "gearscentrifuge", "centrifugeTag" -> Some("centrifuge")))))
+    else None)
+
+    val krakenPage = (if (krakenExecuted) Some("Kraken analysis" -> Future(ReportPage(List(), List(
+      "Krona plot" -> Future(ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp"
+      ))), Map())))
+    else None)
+
+    val qiimeClosedPage = (if (qiimeClosesOtuTable.isDefined) Some("Qiime closed reference analysis" -> Future(ReportPage(List(), List(
+      "Krona plot" -> Future(ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp"
+      ))), Map("biomFile" -> new File(run.outputDir + File.separator + qiimeClosesOtuTable.get.path)))))
+    else None)
+
+    val qiimeOpenPage = (if (qiimeOpenOtuTable.isDefined) Some("Qiime open reference analysis" -> Future(ReportPage(List(), List(
+      "Krona plot" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp"
+      )), Map("biomFile" -> new File(run.outputDir + File.separator + qiimeOpenOtuTable.get.path)))))
+    else None)
+
     ReportPage(
-      (if (centrifugeExecuted) List("Centriguge analysis" -> Future(ReportPage(List("Non-unique" ->
-        Future(ReportPage(List(), List("All mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
-          Map("summaryStatsTag" -> "centrifuge_report")
-        )), Map()))), List(
-        "Unique mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
-          Map("summaryStatsTag" -> "centrifuge_unique_report")
-        )), Map("summaryModuleTag" -> "gearscentrifuge", "centrifugeTag" -> Some("centrifuge")))))
-      else Nil) ::: (if (krakenExecuted) List("Kraken analysis" -> Future(ReportPage(List(), List(
-        "Krona plot" -> Future(ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp"
-        ))), Map())))
-      else Nil) ::: (if (qiimeClosesOtuTable.isDefined) List("Qiime closed reference analysis" -> Future(ReportPage(List(), List(
-        "Krona plot" -> Future(ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp"
-        ))), Map("biomFile" -> new File(run.outputDir + File.separator + qiimeClosesOtuTable.get.path)))))
-      else Nil) ::: (if (qiimeOpenOtuTable.isDefined) List("Qiime open reference analysis" -> Future(ReportPage(List(), List(
-        "Krona plot" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp"
-        )), Map("biomFile" -> new File(run.outputDir + File.separator + qiimeOpenOtuTable.get.path)))))
-      else Nil) ::: List("Samples" -> generateSamplesPage(pageArgs)) ++
-        Map(
-          "Versions" -> Future(ReportPage(List(), List(
-            "Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp")
-          ), Map()))
+      List(centrifugePage, krakenPage, qiimeClosedPage, qiimeOpenPage).flatten ::: List(
+        "Samples" -> generateSamplesPage(pageArgs)
         ),
       List(
         "Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/gearsFront.ssp")) ++

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsSingleReport.scala

@@ -28,6 +28,8 @@ class GearsSingleReport(val parent: Configurable) extends ReportBuilderExtension
 
 object GearsSingleReport extends ReportBuilder {
 
+  def pipelineName = "gearssingle"
+
   override def extFiles = super.extFiles ++ List("js/gears.js", "js/krona-2.0.js", "img/krona/loading.gif", "img/krona/hidden.png", "img/krona/favicon.ico")
     .map(x => ExtFile("/nl/lumc/sasc/biopet/pipelines/gears/report/ext/" + x, x))
 
@@ -39,10 +41,7 @@ object GearsSingleReport extends ReportBuilder {
     val centrifugeExecuted = Await.result(summary.getStatsSize(runId, "gearscentrifuge", "centrifuge_report", sample = sampleId.map(SampleId), library = libId.map(LibraryId)), Duration.Inf) == 1
 
     ReportPage(
-      List(
-        "Versions" -> Future(ReportPage(List(),
-          List(("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"))), Map()))
-      ),
+      Nil,
       List("Gears intro" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/gearsSingleFront.ssp")) ++
         (if (krakenExecuted) List("Kraken analysis" ->
           ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp"))

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MappingReport.scala

@@ -37,6 +37,8 @@ object MappingReport extends ReportBuilder {
   /** Name of report */
   val reportName = "Mapping Report"
 
+  def pipelineName = "mapping"
+
   override def extFiles = super.extFiles ++ List("js/gears.js", "js/krona-2.0.js", "img/krona/loading.gif", "img/krona/hidden.png", "img/krona/favicon.ico")
     .map(x => ExtFile("/nl/lumc/sasc/biopet/pipelines/gears/report/ext/" + x, x))
 
@@ -53,11 +55,7 @@ object MappingReport extends ReportBuilder {
       Some(BammetricsReport.bamMetricsPage(summary, sampleId, libId))
     } else None
     ReportPage((if (skipFlexiprep) Nil else List("QC" -> FlexiprepReport.flexiprepPage)) :::
-      bamMetricsPage.map(_.subPages).getOrElse(Nil) ::: List(
-        "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
-        )), Map())),
-        "Files" -> Future(ReportPage(List(), Nil, Map()))
-      ) :::
+      bamMetricsPage.map(_.subPages).getOrElse(Nil) :::
         (if (krakenExecuted) List("Gears - Metagenomics" -> Future(ReportPage(List(), List(
           "Sunburst analysis" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/gearsSunburst.ssp"
           )), Map())))

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingReport.scala

@@ -79,10 +79,7 @@ trait MultisampleMappingReportTrait extends MultisampleReportBuilder {
         else Nil) ++
         List("Reference" -> Future(ReportPage(List(), List(
           "Reference" -> ReportSection("/nl/lumc/sasc/biopet/core/report/reference.ssp", Map("pipeline" -> pipelineName))
-        ), Map())),
-          "Files" -> filesPage(),
-          "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp"
-          )), Map()))
+        ), Map()))
         ),
       List(
         "Report" -> frontSection) ++
@@ -113,32 +110,6 @@ trait MultisampleMappingReportTrait extends MultisampleReportBuilder {
     )
   }
 
-  /** Files page, can be used general or at sample level */
-  def filesPage(sampleId: Option[Int] = None, libraryId: Option[Int] = None): Future[ReportPage] = {
-    val dbFiles = summary.getFiles(runId, sample = sampleId.map(SampleId),
-      library = libraryId.map(LibraryId))
-      .map(_.groupBy(_.pipelineId))
-    val modulePages = dbFiles.map(_.map {
-      case (pipelineId, files) =>
-        val moduleSections = files.groupBy(_.moduleId).map {
-          case (moduleId, files) =>
-            val moduleName: Future[String] = moduleId match {
-              case Some(id) => summary.getModuleName(pipelineId, id).map(_.getOrElse("Pipeline"))
-              case _        => Future("Pipeline")
-            }
-            moduleName.map(_ -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> files)))
-        }
-        val moduleSectionsSorted = moduleSections.find(_._1 == "Pipeline") ++ moduleSections.filter(_._1 != "Pipeline")
-        summary.getPipelineName(pipelineId = pipelineId).map(_.get -> Future(ReportPage(Nil, Await.result(Future.sequence(moduleSectionsSorted), Duration.Inf).toList, Map())))
-    })
-
-    val pipelineFiles = summary.getPipelineId(runId, pipelineName).flatMap(pipelinelineId => dbFiles.map(x => x(pipelinelineId.get).filter(_.moduleId.isEmpty)))
-
-    modulePages.flatMap(Future.sequence(_)).map(x => ReportPage(x.toList,
-      s"$pipelineName files" -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> Await.result(pipelineFiles, Duration.Inf))) ::
-        "Sub pipelines/modules" -> ReportSection("/nl/lumc/sasc/biopet/core/report/fileModules.ssp", Map("pipelineIds" -> Await.result(dbFiles.map(_.keys.toList), Duration.Inf))) :: Nil, Map()))
-  }
-
   /** Single sample page */
   def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage] = Future {
     val krakenExecuted = Await.result(summary.getStatsSize(runId, "gearskraken", "krakenreport",
@@ -161,9 +132,7 @@ trait MultisampleMappingReportTrait extends MultisampleReportBuilder {
       else Nil) ::: (if (krakenExecuted) List("Dustbin analysis" -> Future(ReportPage(List(), List(
         "Krona Plot" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp"
         )), Map())))
-      else Nil) ++
-      List("Files" -> filesPage(sampleId = sampleId)
-      ), List(
+      else Nil), List(
       "Alignment" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp",
         Map("showPlot" -> true)),
       "Preprocessing" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp", Map("sampleLevel" -> true))) ++
@@ -195,7 +164,7 @@ trait MultisampleMappingReportTrait extends MultisampleReportBuilder {
         else Nil) ::: (if (krakenExecuted) List("Dustbin analysis" -> Future(ReportPage(List(), List(
           "Krona Plot" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp"
           )), Map())))
-        else Nil) ::: List("Files" -> filesPage(sampleId = sampleId, libraryId = libId)),
+        else Nil),
       "Alignment" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp") ::
         (if (flexiprepExecuted) List("QC reads" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepReadSummary.ssp"),
           "QC bases" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepBaseSummary.ssp"))