Added some more join methods for the report

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -23,6 +23,8 @@ import org.broadinstitute.gatk.utils.commandline.Input
 import org.fusesource.scalate.{TemplateEngine, TemplateSource}
 
 import scala.collection.mutable
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
 import scala.language.postfixOps
 
 /**
@@ -69,6 +71,8 @@ trait ReportBuilderExtension extends ToolCommandFunction {
 
 trait ReportBuilder extends ToolCommand {
 
+  implicit def toOption[T](x: T) : Option[T] = Option(x)
+
   case class Args(summaryDbFile: File = null,
                   outputDir: File = null,
                   runId: Int = 0,
@@ -195,6 +199,10 @@ trait ReportBuilder extends ToolCommand {
     logger.info("Parsing summary")
     setSummary = SummaryDb.openSqliteSummary(cmdArgs.summaryDbFile)
     setRunId = cmdArgs.runId
+    setSampleCache = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf).map(r => r.name -> r.id).toMap
+    setLibraryCache = Await.result(summary.getLibraries(runId = Some(runId)), Duration.Inf).map(r => (setSampleCache.find(_._2 == r.sampleId).get._1, r.name) -> r.id).toMap
+    setPipelineCache = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf).map(r => r.name -> r.id).toMap
+    setModuleCache = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf).map(r => r.name -> r.id).toMap
 
     total = ReportBuilder.countPages(indexPage)
     logger.info(total + " pages to be generated")

biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/summary/db/SummaryDb.scala

@@ -76,8 +76,8 @@ class SummaryDb(val db: Database) extends Closeable {
   }
 
   /** Return samplId of a specific runId + sampleName */
-  def getSampleName(sampleId: Int): Future[Option[Int]] = {
-    getSamples(sampleId = Some(sampleId)).map(_.headOption.map(_.id))
+  def getSampleName(sampleId: Int): Future[Option[String]] = {
+    getSamples(sampleId = Some(sampleId)).map(_.headOption.map(_.name))
   }
 
   /** Return sample tags of a specific sample as a map */
@@ -110,6 +110,11 @@ class SummaryDb(val db: Database) extends Closeable {
     getLibraries(runId = Some(runId), sampleId = Some(sampleId), name = Some(name)).map(_.headOption.map(_.id))
   }
 
+  /** Return a libraryId for a specific combination */
+  def getLibraryName(libraryId: Int): Future[Option[String]] = {
+    getLibraries(libId = Some(libraryId)).map(_.headOption.map(_.name))
+  }
+
   /** Return library tags as a map */
   def getLibraryTags(libId: Int): Future[Option[Map[String, Any]]] = {
     db.run(libraries.filter(_.id === libId).map(_.tags).result)
@@ -190,20 +195,45 @@ class SummaryDb(val db: Database) extends Closeable {
 
   /** Return a Query for [[Stats]] */
   def statsFilter(runId: Option[Int] = None, pipelineId: Option[Int] = None, moduleId: Option[Option[Int]] = None,
-                          sampleId: Option[Option[Int]] = None, libId: Option[Option[Int]] = None) = {
+                  sampleId: Option[Option[Int]] = None, libId: Option[Option[Int]] = None,
+                  mustHaveSample: Boolean = false, mustHaveLibrary:Boolean = false) = {
     var f: Query[Stats, Stats#TableElementType, Seq] = stats
     runId.foreach(r => f = f.filter(_.runId === r))
     pipelineId.foreach(r => f = f.filter(_.pipelineId === r))
     moduleId.foreach(r => f = if (r.isDefined) f.filter(_.moduleId === r.get) else f.filter(_.moduleId.isEmpty))
     sampleId.foreach(r => f = if (r.isDefined) f.filter(_.sampleId === r.get) else f.filter(_.sampleId.isEmpty))
     libId.foreach(r => f = if (r.isDefined) f.filter(_.libraryId === r.get) else f.filter(_.libraryId.isEmpty))
+    if (mustHaveSample) f = f.filter(_.sampleId.nonEmpty)
+    if (mustHaveLibrary) f = f.filter(_.libraryId.nonEmpty)
+    f
+  }
+
+  def statsJoinFilter(runId: Option[Int] = None, pipelineName: Option[String] = None, moduleName: Option[Option[String]] = None,
+                      sampleName: Option[Option[String]] = None, libraryName: Option[Option[String]] = None,
+                      mustHaveSample: Boolean = false, mustHaveLibrary:Boolean = false) = {
+    var f: Query[Stats, Stats#TableElementType, Seq] = stats
+    if (runId.isDefined) f = f.filter(_.runId === runId.get)
+    if (pipelineName.isDefined) f = f.join(pipelines).on(_.pipelineId === _.id).filter(_._2.name === pipelineName.get).map(_._1)
+    moduleName.foreach(r => if (r.isDefined) f = f.join(modules).on(_.moduleId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.moduleId.isEmpty))
+    sampleName.foreach(r => if (r.isDefined) f = f.join(samples).on(_.sampleId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.sampleId.isEmpty))
+    libraryName.foreach(r => if (r.isDefined) f = f.join(libraries).on(_.libraryId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.libraryId.isEmpty))
+    if (mustHaveSample) f = f.filter(_.sampleId.nonEmpty)
+    if (mustHaveLibrary) f = f.filter(_.libraryId.nonEmpty)
     f
   }
 
   /** Return all stats that match given criteria */
   def getStats(runId: Option[Int] = None, pipelineId: Option[Int] = None, moduleId: Option[Option[Int]] = None,
-               sampleId: Option[Option[Int]] = None, libId: Option[Option[Int]] = None): Future[Seq[Stat]] = {
-    db.run(statsFilter(runId, pipelineId, moduleId, sampleId, libId).result)
+               sampleId: Option[Option[Int]] = None, libId: Option[Option[Int]] = None,
+               mustHaveSample: Boolean = false, mustHaveLibrary:Boolean = false): Future[Seq[Stat]] = {
+    db.run(statsFilter(runId, pipelineId, moduleId, sampleId, libId, mustHaveSample, mustHaveLibrary).result)
+  }
+
+  /** Return number of results */
+  def getStatsSize(runId: Option[Int] = None, pipelineName: Option[String] = None, moduleName: Option[Option[String]] = None,
+                   sampleName: Option[Option[String]] = None, libName: Option[Option[String]] = None,
+                   mustHaveSample: Boolean = false, mustHaveLibrary:Boolean = false): Future[Int] = {
+    db.run(statsJoinFilter(runId, pipelineName, moduleName, sampleName, libName, mustHaveSample, mustHaveLibrary).size.result)
   }
 
   /** Get a single stat as [[Map[String, Any]] */
@@ -267,6 +297,21 @@ class SummaryDb(val db: Database) extends Closeable {
     f
   }
 
+  def filesJoinFilter(runId: Option[Int] = None, pipelineName: Option[String] = None, moduleName: Option[Option[String]] = None,
+                      sampleName: Option[Option[String]] = None, libraryName: Option[Option[String]] = None,
+                      mustHaveSample: Boolean = false, mustHaveLibrary:Boolean = false, key: Option[String] = None) = {
+    var f: Query[Files, Files#TableElementType, Seq] = files
+    if (runId.isDefined) f = f.filter(_.runId === runId.get)
+    if (key.isDefined) f = f.filter(_.key === key.get)
+    if (pipelineName.isDefined) f = f.join(pipelines).on(_.pipelineId === _.id).filter(_._2.name === pipelineName.get).map(_._1)
+    moduleName.foreach(r => if (r.isDefined) f = f.join(modules).on(_.moduleId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.moduleId.isEmpty))
+    sampleName.foreach(r => if (r.isDefined) f = f.join(samples).on(_.sampleId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.sampleId.isEmpty))
+    libraryName.foreach(r => if (r.isDefined) f = f.join(libraries).on(_.libraryId === _.id).filter(_._2.name === r.get).map(_._1) else f = f.filter(_.libraryId.isEmpty))
+    if (mustHaveSample) f = f.filter(_.sampleId.nonEmpty)
+    if (mustHaveLibrary) f = f.filter(_.libraryId.nonEmpty)
+    f
+  }
+
   /** Returns all [[Files]] with the given criteria */
   def getFiles(runId: Option[Int] = None, pipelineId: Option[Int] = None, moduleId: Option[Option[Int]],
                sampleId: Option[Option[Int]] = None, libId: Option[Option[Int]] = None,
@@ -274,6 +319,10 @@ class SummaryDb(val db: Database) extends Closeable {
     db.run(filesFilter(runId, pipelineId, moduleId, sampleId, libId, key).result)
   }
 
+  def getFile(runId: Int, pipelineName: String, moduleName: Option[String], sampleName: Option[String], libraryName: Option[String], key: String): Future[Option[File]] = {
+    db.run(filesJoinFilter(Some(runId), Some(pipelineName), Some(moduleName), Some(sampleName), Some(libraryName), key = Some(key)).map(_.path).result).map(_.headOption.map(new File(_)))
+  }
+
   /** Creates a file. This method will raise expection if it already exist */
   def createFile(runId: Int, pipelineId: Int, moduleId: Option[Int] = None,
                  sampleId: Option[Int] = None, libId: Option[Int] = None,

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsReport.scala

@@ -16,10 +16,13 @@ package nl.lumc.sasc.biopet.pipelines.gears
 
 import java.io.File
 
-import nl.lumc.sasc.biopet.core.report.{ ReportSection, ReportPage, MultisampleReportBuilder, ReportBuilderExtension }
+import nl.lumc.sasc.biopet.core.report.{MultisampleReportBuilder, ReportBuilderExtension, ReportPage, ReportSection}
 import nl.lumc.sasc.biopet.pipelines.flexiprep.FlexiprepReport
 import nl.lumc.sasc.biopet.utils.config.Configurable
 
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
 /**
  * Report for Gears
  *
@@ -37,12 +40,10 @@ object GearsReport extends MultisampleReportBuilder {
     .map(x => ExtFile("/nl/lumc/sasc/biopet/pipelines/gears/report/ext/" + x, x))
 
   def indexPage = {
-    val krakenExecuted = summary.getSampleValues("gearskraken", "stats", "krakenreport").values.forall(_.isDefined)
-    val centrifugeExecuted = summary.getSampleValues("gearscentrifuge", "stats", "centrifuge_report").values.forall(_.isDefined)
-    val qiimeClosesOtuTable = summary.getValue("gears", "files", "pipeline", "qiime_closed_otu_table", "path")
-      .map(x => new File(x.toString))
-    val qiimeOpenOtuTable = summary.getValue("gears", "files", "pipeline", "qiime_open_otu_table", "path")
-      .map(x => new File(x.toString))
+    val krakenExecuted = Await.result(summary.getStatsSize(runId, "gearskraken", Some("krakenreport"), libName = None, mustHaveSample = true), Duration.Inf) >= sampleCache.size
+    val centrifugeExecuted = Await.result(summary.getStatsSize(runId, "gearscentrifuge", Some("centrifuge_report"), libName = None, mustHaveSample = true), Duration.Inf) >= sampleCache.size
+    val qiimeClosesOtuTable = Await.result(summary.getFile(runId, "gears", None, None, None, key = "qiime_closed_otu_table"), Duration.Inf)
+    val qiimeOpenOtuTable = Await.result(summary.getFile(runId, "gears", None, None, None, key = "qiime_open_otu_table"), Duration.Inf)
 
     ReportPage(
       (if (centrifugeExecuted) List("Centriguge analysis" -> ReportPage(List("Non-unique" -> ReportPage(List(), List("All mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
@@ -79,13 +80,13 @@ object GearsReport extends MultisampleReportBuilder {
   }
 
   /** Single sample page */
-  def samplePage(sampleId: String, args: Map[String, Any]): ReportPage = {
-    val krakenExecuted = summary.getValue(Some(sampleId), None, "gearskraken", "stats", "krakenreport").isDefined
-    val centrifugeExecuted = summary.getValue(Some(sampleId), None, "gearscentrifuge", "stats", "centrifuge_report").isDefined
-    val qiimeClosesOtuTable = summary.getValue(Some(sampleId), None, "gearsqiimeclosed", "files", "pipeline", "otu_table", "path")
-      .map(x => new File(x.toString))
-    val qiimeOpenOtuTable = summary.getValue(Some(sampleId), None, "gearsqiimeopen", "files", "pipeline", "otu_table", "path")
-      .map(x => new File(x.toString))
+  def samplePage(sampleId: Int, args: Map[String, Any]): ReportPage = {
+    val sampleName = Await.result(summary.getSampleName(sampleId), Duration.Inf)
+    val krakenExecuted = Await.result(summary.getStatsSize(runId, "gearskraken", Some("krakenreport"), sampleName = sampleName, libName = None), Duration.Inf) == 1
+    val centrifugeExecuted = Await.result(summary.getStatsSize(runId, "gearscentrifuge", Some("centrifuge_report"), sampleName = sampleName, libName = None), Duration.Inf) == 1
+    val qiimeClosesOtuTable = Await.result(summary.getFile(runId, "gears", None, sampleName, None, key = "qiime_closed_otu_table"), Duration.Inf)
+    val qiimeOpenOtuTable = Await.result(summary.getFile(runId, "gears", None, sampleName, None, key = "qiime_open_otu_table"), Duration.Inf)
+
 
     ReportPage((if (centrifugeExecuted) List("Centriguge analysis" -> ReportPage(List("Non-unique" -> ReportPage(List(), List("All mappings" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
       Map("summaryStatsTag" -> "centrifuge_report")
@@ -110,14 +111,17 @@ object GearsReport extends MultisampleReportBuilder {
   }
 
   /** Library page */
-  def libraryPage(sampleId: String, libId: String, args: Map[String, Any]): ReportPage = {
-    val flexiprepExecuted = summary.getLibraryValue(sampleId, libId, "flexiprep").isDefined
-    val krakenExecuted = summary.getValue(Some(sampleId), Some(libId), "gearskraken", "stats", "krakenreport").isDefined
-    val centrifugeExecuted = summary.getValue(Some(sampleId), Some(libId), "gearscentrifuge", "stats", "centrifuge_report").isDefined
-    val qiimeClosesOtuTable = summary.getValue(Some(sampleId), Some(libId), "gearsqiimeclosed", "files", "pipeline", "otu_table", "path")
-      .map(x => new File(x.toString))
-    val qiimeOpenOtuTable = summary.getValue(Some(sampleId), Some(libId), "gearsqiimeopen", "files", "pipeline", "otu_table", "path")
-      .map(x => new File(x.toString))
+  def libraryPage(sampleId: Int, libId: Int, args: Map[String, Any]): ReportPage = {
+    val sName = Await.result(summary.getSampleName(sampleId), Duration.Inf)
+    val lName = Await.result(summary.getLibraryName(libId), Duration.Inf)
+
+    val flexiprepExecuted = Await.result(summary.getStatsSize(Some(runId), Some("flexiprep"), None, Some(sName),Some(lName)), Duration.Inf) >= 1
+
+    val krakenExecuted = Await.result(summary.getStatsSize(runId, "gearskraken", Some("krakenreport"), sampleName = sName, libName = lName), Duration.Inf) == 1
+    val centrifugeExecuted = Await.result(summary.getStatsSize(runId, "gearscentrifuge", Some("centrifuge_report"), sampleName = sName, libName = lName), Duration.Inf) == 1
+    val qiimeClosesOtuTable = Await.result(summary.getFile(runId, "gears", None, sName, lName, key = "qiime_closed_otu_table"), Duration.Inf)
+    val qiimeOpenOtuTable = Await.result(summary.getFile(runId, "gears", None, sName, lName, key = "qiime_open_otu_table"), Duration.Inf)
+
 
     ReportPage(
       (if (flexiprepExecuted) List("QC" -> FlexiprepReport.flexiprepPage) else Nil

gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsSingleReport.scala

@@ -17,6 +17,9 @@ package nl.lumc.sasc.biopet.pipelines.gears
 import nl.lumc.sasc.biopet.core.report._
 import nl.lumc.sasc.biopet.utils.config.Configurable
 
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
 class GearsSingleReport(val parent: Configurable) extends ReportBuilderExtension {
   def builder = GearsSingleReport
 }
@@ -27,8 +30,9 @@ object GearsSingleReport extends ReportBuilder {
     .map(x => ExtFile("/nl/lumc/sasc/biopet/pipelines/gears/report/ext/" + x, x))
 
   def indexPage = {
-    val krakenExecuted = summary.getValue(sampleId, libId, "gearskraken", "stats", "krakenreport").isDefined
-    val centrifugeExecuted = summary.getValue(sampleId, libId, "gearscentrifuge", "stats", "centrifuge_report").isDefined
+    Await.result(summary.getStatsSize(runId, "gearscentrifuge", Some("centrifuge_report"), sampleId, libId), Duration.Inf) == 1
+    val krakenExecuted = Await.result(summary.getStatsSize(runId, "gearskraken", Some("krakenreport"), sampleId, libId), Duration.Inf) == 1
+    val centrifugeExecuted = Await.result(summary.getStatsSize(runId, "gearscentrifuge", Some("centrifuge_report"), sampleId, libId), Duration.Inf) == 1
 
     ReportPage(
       List(