Merge remote-tracking branch 'origin/develop' into fix-BIOPET-306-2

# Conflicts:
#	biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/rscript/LinePlot.scala
#	shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaReport.scala

bam2wig/pom.xml

@@ -44,18 +44,6 @@
             <artifactId>BiopetExtensions</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_2.10</artifactId>
-            <version>2.2.1</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <version>6.8</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file

bam2wig/src/main/scala/nl/lumc/sasc/biopet/pipelines/bamtobigwig/Bam2Wig.scala

@@ -27,7 +27,7 @@ import org.broadinstitute.gatk.queue.QScript
  *
  * Created by pjvan_thof on 1/29/15.
  */
-class Bam2Wig(val root: Configurable) extends QScript with BiopetQScript {
+class Bam2Wig(val parent: Configurable) extends QScript with BiopetQScript {
   def this() = this(null)
 
   @Input(doc = "Input bam file", required = true)

bam2wig/src/main/scala/nl/lumc/sasc/biopet/pipelines/bamtobigwig/BamToChromSizes.scala

@@ -28,7 +28,7 @@ import scala.collection.JavaConversions._
  *
  * Created by pjvan_thof on 1/29/15.
  */
-class BamToChromSizes(val root: Configurable) extends InProcessFunction with Configurable {
+class BamToChromSizes(val parent: Configurable) extends InProcessFunction with Configurable {
   @Input
   var bamFile: File = _
 

bammetrics/pom.xml

@@ -42,17 +42,5 @@
             <artifactId>BiopetToolsExtensions</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <version>6.8</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_2.10</artifactId>
-            <version>2.2.1</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

bammetrics/src/main/resources/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp

-#import(nl.lumc.sasc.biopet.utils.summary.Summary)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
 #import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
 #import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
 #import(java.io.File)
-<%@ var summary: Summary %>
-<%@ var sampleId: Option[String] = None %>
-<%@ var libId: Option[String] = None %>
+<%@ var summary: SummaryDb %>
+<%@ var sampleId: Option[Int] = None %>
+<%@ var libId: Option[Int] = None %>
 <%@ var sampleLevel: Boolean = false %>
 <%@ var rootPath: String %>
 <%@ var outputDir: File %>
 <%@ var showPlot: Boolean = false %>
 <%@ var showTable: Boolean = true %>
 <%@ var showIntro: Boolean = true%>
+<%@ var runId: Int %>
 #{
-    val samples = sampleId match {
-        case Some(sample) => {
-            List(sample.toString)
-        }
-        case _ => summary.samples.toList
-    }
+    val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
 }#
 #if (showIntro)
     <br/>
@@ -69,24 +69,31 @@
         <th>(%)</th>
     </tr></thead>
     <tbody>
-        #for (sample <- samples.toList.sorted)
+        #for (sample <- samples.sortBy(_.name))
             #{
-                val libs = (libId, sampleLevel) match {
-                case (_, true) => List("")
-                case (Some(libId), _) => List(libId.toString).sorted
-                case _ => summary.libraries(sample).toList.sorted
+                val libs: List[Option[Int]] = (libId, sampleLevel) match {
+                case (_, true) => List(None)
+                case (Some(_), _) => List(libId)
+                case _ => Await.result(summary.getLibraries(sampleId = Some(sample.id), runId = Some(runId)), Duration.Inf).map(x => Some(x.id)).toList
                 }
             }#
-            <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
+            <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
             #for (libId <- libs)
+                #{ val libName = libId.map(l => Await.result(summary.getLibraryName(l), Duration.Inf)) }#
                 #if (libs.head != libId) <tr> #end
-                    #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
+                    #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
                     #{
-                        val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
-                        val total = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "All")):_*).getOrElse(0L).asInstanceOf[Long]
-                        val mapped = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long]
-                        val duplicates = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long]
-                        val secondary = summary.getValue((prefixPath ::: List("bamstats", "flagstats", "NotPrimaryAlignment")):_*).getOrElse(0L).asInstanceOf[Long]
+                        val statsPaths = Map(
+                        "All" -> List("flagstats", "All"),
+                        "Mapped" -> List("flagstats", "Mapped"),
+                        "Duplicates" -> List("flagstats", "Duplicates"),
+                        "NotPrimaryAlignment" -> List("flagstats", "NotPrimaryAlignment")
+                        )
+                        val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
+                        val total = results("All").getOrElse(0L).asInstanceOf[Long]
+                        val mapped = results("Mapped").getOrElse(0L).asInstanceOf[Long]
+                        val duplicates = results("Duplicates").getOrElse(0L).asInstanceOf[Long]
+                        val secondary = results("NotPrimaryAlignment").getOrElse(0L).asInstanceOf[Long]
                     }#
                     <td>${total}</td>
                     <td>${mapped}</td>

bammetrics/src/main/resources/nl/lumc/sasc/biopet/pipelines/bammetrics/clipping.ssp

-#import(nl.lumc.sasc.biopet.utils.summary.Summary)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
 #import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
 #import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
 #import(java.io.File)
-#import(org.apache.commons.io.FileUtils)
-<%@ var summary: Summary %>
-<%@ var sampleId: Option[String] = None %>
-<%@ var libId: Option[String] = None %>
-<%@ var rootPath: String %>
-<%@ var metricsTag: String = "bammetrics" %>
+<%@ var summary: SummaryDb %>
+<%@ var sampleId: Option[Int] = None %>
+<%@ var libId: Option[Int] = None %>
 <%@ var sampleLevel: Boolean = false %>
+<%@ var rootPath: String %>
 <%@ var outputDir: File %>
-<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
 <%@ var showPlot: Boolean = false %>
 <%@ var showTable: Boolean = true %>
 <%@ var showIntro: Boolean = true%>
+<%@ var runId: Int %>
+<%@ var fields: List[String] = List("min", "max", "mean", "median", "modal")%>
 #{
-    val samples = sampleId match {
-    case Some(sample) => {
-    List(sample.toString)
-    }
-    case _ => summary.samples.toList
-    }
+    val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
 }#
 
 #if (showIntro)
@@ -42,7 +40,7 @@
 #end
 
 #if (showPlot)
-    #{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libId = libId) }#
+    #{ BammetricsReport.clippingPlot(outputDir, "clipping", summary, !sampleLevel, sampleId = sampleId, libraryId = libId) }#
 
     <div class="panel-body">
     <img src="clipping.png" class="img-responsive" />
@@ -71,27 +69,25 @@
         #end
     </tr></thead>
     <tbody>
-        #for (sample <- samples.toList.sorted)
+        #for (sample <- samples.sortBy(_.name))
             #{
-                val libs = (libId, sampleLevel) match {
-                case (_, true) => List("")
-                case (Some(libId), _) => List(libId.toString)
-                case _ => summary.libraries(sample).toList
+                val libs: List[Option[Int]] = (libId, sampleLevel) match {
+                case (_, true) => List(None)
+                case (Some(_), _) => List(libId)
+                case _ => Await.result(summary.getLibraries(sampleId = Some(sample.id), runId = Some(runId)), Duration.Inf).map(x => Some(x.id)).toList
                 }
             }#
-            <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
+            <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
             #for (libId <- libs)
+                #{ val libName = libId.map(l => Await.result(summary.getLibraryName(l), Duration.Inf)) }#
                 #if (libs.head != libId) <tr> #end
-                    #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
+                    #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample.name}/Libraries/${libName}/index.html">${libName}</a></td> #end
                     #{
-                        val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List("bammetrics", "stats")
-
-                        val fieldValues = for (field <- fields) yield {
-                            summary.getValue((prefixPath ::: List("bamstats", "clipping", "general", field)):_*).getOrElse("N/A")
-                        }
+                        val statsPaths = fields.map(x => x -> List("clipping", "general", x)).toMap
+                        val results = summary.getStatKeys(runId, "bammetrics", "bamstats", sample = sample.id, library = libId.map(LibraryId).getOrElse(NoLibrary), keyValues = statsPaths)
                     }#
-                    #for (value <- fieldValues)
-                        <td>${value}</td>
+                    #for (field <- fields)
+                        <td>${results(field)}</td>
                     #end
                 </tr>
             #end

bammetrics/src/main/resources/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsMultiTable.ssp

 #import(nl.lumc.sasc.biopet.utils.IoUtils)
-#import(nl.lumc.sasc.biopet.utils.summary.Summary)
-#import(nl.lumc.sasc.biopet.core.report.ReportPage)
 #import(org.apache.commons.io.FileUtils)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
+#import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
+#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
 #import(java.io.File)
-<%@ var summary: Summary %>
-<%@ var sampleId: Option[String] = None %>
-<%@ var libId: Option[String] = None %>
+<%@ var summary: SummaryDb %>
+<%@ var sampleId: Option[Int] = None %>
 <%@ var sampleLevel: Boolean = true %>
-<%@ var rootPath: String %>
+<%@ var libId: Option[Int] = None %>
 <%@ var outputDir: File %>
+<%@ var rootPath: String %>
 <%@ var metricsTag: String = "bammetrics" %>
 <%@ var target: Option[String] %>
+<%@ var runId: Int %>
+<%@ var fields: List[String] = List("mean", "median", "max", "horizontal", "frac_min_10x", "frac_min_20x", "frac_min_30x", "frac_min_40x", "frac_min_50x") %>
 #{
-    val samples = sampleId match {
-        case Some(sample) => List(sample.toString)
-        case _ => summary.samples.toList
-    }
+    val samples = Await.result(summary.getSamples(runId = Some(runId)), Duration.Inf)
 }#
 
 
 <table class="table">
 <thead><tr>
     <th>sample</th>
-    <th>mean</th>
-    <th>median</th>
-    <th>max</th>
-    <th>horizontal</th>
-    <th>frac min 10x</th>
-    <th>frac min 20x</th>
-    <th>frac min 30x</th>
-    <th>frac min 40x</th>
-    <th>frac min 50x</th>
+    #for (field <- fields)
+        <th>${field}</th>
+    #end
 </tr></thead>
 <tbody>
-    #for (sample <- samples.toList.sorted)
+    #for (sample <- samples.sortBy(_.name))
         #{
-            val libs = (libId, sampleLevel) match {
-            case (_, true) => List("")
-            case (Some(libId), _) => List(libId.toString)
-            case _ => summary.libraries(sample).toList
+            val libs: List[Option[Int]] = (libId, sampleLevel) match {
+                case (_, true) => List(None)
+                case (Some(_), _) => List(libId)
+                case _ => Await.result(summary.getLibraries(sampleId = Some(sample.id), runId = Some(runId)), Duration.Inf).map(x => Some(x.id)).toList
             }
         }#
-        <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample}/index.html">${sample}</a></td>
+        <tr><td rowspan="${libs.size}"><a href="${rootPath}Samples/${sample.name}/index.html">${sample.name}</a></td>
         #for (libId <- libs)
             #if (libs.head != libId) <tr> #end
-                #if (!sampleLevel) <td><a href="${rootPath}Samples/${sample}/Libraries/${libId}/index.html">${libId}</a></td> #end
                 #{
-                    val prefixPath = List("samples", sample) ::: (if (libId.isEmpty) Nil else List("libraries", libId)) ::: List(metricsTag, "stats", target.get + "_cov_stats", "coverage", "_all")
-                    val total = summary.getValue((prefixPath ::: List("biopet_flagstat", "All")):_*).getOrElse(0L).asInstanceOf[Long]
-                    val mapped = summary.getValue((prefixPath ::: List("biopet_flagstat", "Mapped")):_*).getOrElse(0L).asInstanceOf[Long]
-                    val duplicates = summary.getValue((prefixPath ::: List("biopet_flagstat", "Duplicates")):_*).getOrElse(0L).asInstanceOf[Long]
+                    val moduleName = target.get + "_cov_stats"
+                    val statsPaths = fields.map(x => x -> List("coverage", "_all", x)).toMap
+                    val values = summary.getStatKeys(runId, PipelineName(metricsTag), ModuleName(moduleName), SampleId(sample.id), libId.map(LibraryId).getOrElse(NoLibrary), statsPaths)
                 }#
-                <td>${summary.getValue((prefixPath ::: "mean" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "median" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "max" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "horizontal" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "frac_min_10x" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "frac_min_20x" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "frac_min_30x" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "frac_min_40x" :: Nil):_*)}</td>
-                <td>${summary.getValue((prefixPath ::: "frac_min_50x" :: Nil):_*)}</td>
+                #for (field <- fields)
+                    <th>${values(field)}</th>
+                #end
             </tr>
         #end
     #end

bammetrics/src/main/resources/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp

 #import(nl.lumc.sasc.biopet.utils.IoUtils)
-#import(nl.lumc.sasc.biopet.utils.summary.Summary)
-#import(nl.lumc.sasc.biopet.core.report.ReportPage)
 #import(org.apache.commons.io.FileUtils)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._)
+#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._)
+#import(nl.lumc.sasc.biopet.core.report.ReportPage)
+#import(scala.concurrent.Await)
+#import(scala.concurrent.duration.Duration)
+#import(nl.lumc.sasc.biopet.pipelines.bammetrics.BammetricsReport)
 #import(java.io.File)
-<%@ var summary: Summary %>
-<%@ var sampleId: Option[String] %>
-<%@ var libId: Option[String] = None %>
+<%@ var summary: SummaryDb %>
+<%@ var sampleId: Option[Int] %>
+<%@ var libId: Option[Int] = None %>
 <%@ var outputDir: File %>
 <%@ var metricsTag: String = "bammetrics" %>
 <%@ var target: Option[String] %>
+<%@ var runId: Int %>
+<%@ var fields: List[String] = List("mean", "median", "max", "horizontal", "frac_min_10x", "frac_min_20x", "frac_min_30x", "frac_min_40x", "frac_min_50x") %>
 #{
-    val originalPlot = new File(summary.getValue(sampleId, libId, metricsTag, "files", target.get + "_cov_stats", "plot", "path")
-        .getOrElse(throw new IllegalArgumentException("No plot found in summary")).toString)
+    val moduleName = target.get + "_cov_stats"
+
+    val statsPaths = fields.map(x => x -> List("coverage", "_all", x)).toMap
+    val plotFile = Await.result(summary.getFile(runId, PipelineName(metricsTag), ModuleName(moduleName), sampleId.map(SampleId).get, libId.map(LibraryId).getOrElse(NoLibrary), "plot"), Duration.Inf)
+    val originalPlot = new File(plotFile.get.path)
     val plot = new File(outputDir, target.get + "_cov_stats.png")
 
-    val values = summary.getValue(sampleId, libId, metricsTag, "stats", target.get + "_cov_stats", "coverage", "_all")
-        .getOrElse(throw new IllegalArgumentException("No plot found in summary")).asInstanceOf[Map[String, Any]]
+    val values = summary.getStatKeys(runId, PipelineName(metricsTag), ModuleName(moduleName), sampleId.map(SampleId).get, libId.map(LibraryId).getOrElse(NoLibrary), statsPaths)
 
     if (originalPlot.exists()) IoUtils.copyFile(originalPlot, plot)
 }#
-
 <img src="${plot.getName}">
 
 <table class="table">
 <thead><tr>
-    <th>mean</th>
-    <th>median</th>
-    <th>max</th>
-    <th>horizontal</th>
-    <th>frac min 10x</th>
-    <th>frac min 20x</th>
-    <th>frac min 30x</th>
-    <th>frac min 40x</th>
-    <th>frac min 50x</th>
+#for (field <- fields)
+    <th>${field}</th>
+#end
 </tr></thead>
 <tbody>
 <tr>