Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
B
biopet.biopet
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
Mirrors
biopet.biopet
Commits
625bb514
Commit
625bb514
authored
Apr 10, 2017
by
Peter van 't Hof
Committed by
GitHub
Apr 10, 2017
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #57 from biopet/fix-BIOPET-618
Making the report builder complete future proof
parents
abda4ca3
ea77da34
Changes
22
Hide whitespace changes
Inline
Side-by-side
Showing
22 changed files
with
563 additions
and
294 deletions
+563
-294
bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala
...c/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala
+19
-21
biopet-core/src/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp
...c/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp
+20
-13
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala
...mc/sasc/biopet/core/report/MultisampleReportBuilder.scala
+21
-23
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala
...scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala
+125
-51
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportPage.scala
...in/scala/nl/lumc/sasc/biopet/core/report/ReportPage.scala
+3
-1
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala
...ala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala
+1
-0
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilderTest.scala
...asc/biopet/core/report/MultisampleReportBuilderTest.scala
+13
-5
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala
...a/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala
+25
-19
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala
...a/nl/lumc/sasc/biopet/core/summary/WriteSummaryTest.scala
+1
-0
biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/RegionAfCount.scala
.../main/scala/nl/lumc/sasc/biopet/tools/RegionAfCount.scala
+4
-2
biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/bamstats/Histogram.scala
.../scala/nl/lumc/sasc/biopet/tools/bamstats/Histogram.scala
+3
-1
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/process/AsyncProcess.scala
...cala/nl/lumc/sasc/biopet/utils/process/AsyncProcess.scala
+178
-0
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/rscript/Rscript.scala
...ain/scala/nl/lumc/sasc/biopet/utils/rscript/Rscript.scala
+14
-5
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/summary/db/SummaryDb.scala
...cala/nl/lumc/sasc/biopet/utils/summary/db/SummaryDb.scala
+14
-7
biopet-utils/src/test/scala/nl/lumc/sasc/biopet/utils/summary/db/SummaryDbTest.scala
.../nl/lumc/sasc/biopet/utils/summary/db/SummaryDbTest.scala
+2
-0
flexiprep/src/main/resources/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepFastQcPlot.ssp
...c/sasc/biopet/pipelines/flexiprep/flexiprepFastQcPlot.ssp
+4
-2
flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/FlexiprepReport.scala
...umc/sasc/biopet/pipelines/flexiprep/FlexiprepReport.scala
+7
-12
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsReport.scala
...ala/nl/lumc/sasc/biopet/pipelines/gears/GearsReport.scala
+55
-46
gears/src/main/scala/nl/lumc/sasc/biopet/pipelines/gears/GearsSingleReport.scala
.../lumc/sasc/biopet/pipelines/gears/GearsSingleReport.scala
+7
-8
mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MappingReport.scala
...nl/lumc/sasc/biopet/pipelines/mapping/MappingReport.scala
+9
-11
mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMappingReport.scala
...c/biopet/pipelines/mapping/MultisampleMappingReport.scala
+28
-57
shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaReport.scala
...ala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaReport.scala
+10
-10
No files found.
bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala
View file @
625bb514
...
...
@@ -24,8 +24,7 @@ import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb._
import
scala.concurrent.ExecutionContext.Implicits.global
import
scala.concurrent.Await
import
scala.concurrent.
{
Await
,
Future
}
import
scala.concurrent.duration.Duration
class
BammetricsReport
(
val
parent
:
Configurable
)
extends
ReportBuilderExtension
{
...
...
@@ -42,28 +41,27 @@ object BammetricsReport extends ReportBuilder {
/** Name of report */
val
reportName
=
"Bam Metrics"
def
pipelineName
=
"bammetrics"
/** Root page for single BamMetrcis report */
def
indexPage
=
{
val
bamMetricsPage
=
this
.
bamMetricsPage
(
summary
,
sampleId
,
libId
)
ReportPage
(
bamMetricsPage
.
subPages
:::
List
(
"Versions"
->
ReportPage
(
List
(),
List
(
"Executables"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/executables.ssp"
)),
Map
(
)),
"Files"
->
ReportPage
(
List
(),
List
(),
Map
()
)
),
List
(
"Report"
->
ReportSection
(
"/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp"
)
)
:::
bamMetricsPage
.
sections
,
Map
()
)
}
def
indexPage
:
Future
[
ReportPage
]
=
bamMetricsPage
(
summary
,
sampleId
,
libId
).
map
{
bamMetricsPage
=>
ReportPage
(
bamMetricsPage
.
subPages
:::
List
(
"Versions"
->
Future
(
ReportPage
(
List
(),
List
(
"Executables"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/executables.ssp"
)),
Map
()
)),
"Files"
->
filesPage
(
sampleId
,
libId
)
),
List
(
"Report"
->
ReportSection
(
"/nl/lumc/sasc/biopet/pipelines/bammetrics/bamMetricsFront.ssp"
)
)
:::
bamMetricsPage
.
sections
,
Map
()
)
}
/** Generates a page with alignment stats */
def
bamMetricsPage
(
summary
:
SummaryDb
,
sampleId
:
Option
[
Int
],
libId
:
Option
[
Int
],
metricsTag
:
String
=
"bammetrics"
)
=
{
//val pipelineId: Int = summary.getPipelineId(runId, metricsTag).map(_.get)
metricsTag
:
String
=
"bammetrics"
)
:
Future
[
ReportPage
]
=
Future
{
val
wgsExecuted
=
summary
.
getStatsSize
(
runId
,
metricsTag
,
"wgs"
,
sample
=
sampleId
.
map
(
SampleId
),
library
=
libId
.
map
(
LibraryId
))
>=
1
val
rnaExecuted
=
summary
.
getStatsSize
(
runId
,
metricsTag
,
"rna"
,
sample
=
sampleId
.
map
(
SampleId
),
library
=
libId
.
map
(
LibraryId
))
>=
1
...
...
@@ -85,10 +83,10 @@ object BammetricsReport extends ReportBuilder {
ReportPage
(
if
(
targets
.
isEmpty
)
List
()
else
List
(
"Targets"
->
ReportPage
(
else
List
(
"Targets"
->
Future
.
successful
(
ReportPage
(
List
(),
targets
.
map
(
t
=>
t
->
ReportSection
(
"/nl/lumc/sasc/biopet/pipelines/bammetrics/covstatsPlot.ssp"
,
Map
(
"target"
->
Some
(
t
)))),
Map
())),
Map
()))
)
,
List
(
"Summary"
->
ReportSection
(
"/nl/lumc/sasc/biopet/pipelines/bammetrics/alignmentSummary.ssp"
),
"Mapping Quality"
->
ReportSection
(
"/nl/lumc/sasc/biopet/pipelines/bammetrics/mappingQuality.ssp"
,
Map
(
"showPlot"
->
true
)),
...
...
@@ -161,7 +159,7 @@ object BammetricsReport extends ReportBuilder {
plot
.
output
=
pngFile
plot
.
ylabel
=
Some
(
"Reads"
)
plot
.
width
=
Some
(
200
+
(
results
.
size
*
10
))
plot
.
title
=
Some
(
"Aligned
reads"
)
plot
.
title
=
Some
(
"Aligned
_
reads"
)
plot
.
runLocal
()
}
...
...
biopet-core/src/main/resources/nl/lumc/sasc/biopet/core/report/main.ssp
View file @
625bb514
#import(nl.lumc.sasc.biopet.utils.summary.db.SummaryDb)
#import(nl.lumc.sasc.biopet.core.report.ReportPage)
#import(scala.concurrent.Await)
#import(scala.concurrent.duration.Duration)
#import(scala.concurrent.Future)
<
%@
var
summary:
SummaryDb
%
>
<
%@
var
indexPage:
ReportPage
%
>
<
%@
var
indexPage:
Future
[
ReportPage
]
%
>
<
%@
var
reportName:
String
%
>
<
%@
var
page:
ReportPage
%
>
<
%@
var
path:
List
[
String
]
%
>
<
%@
var
args:
Map
[
String
,
Any
]
%
>
<
%@
var
rootPath:
String
%
>
#{
val iPage = Await.result(indexPage, Duration.Inf)
def createMenu(page: ReportPage, path: List[String] = Nil, first: Boolean = true): String = {
val buffer: StringBuffer = new StringBuffer()
...
...
@@ -15,11 +20,13 @@
buffer.append("
<ul
class=
\"dropdown-menu
list-group
\"
>
")
}
for (subPage
<-
page.subPages.sortBy
(
_._1
))
{
for (subPage
Future
<-
page.subPages.sortBy
(
_._1
))
{
val
subPageName =
subPageFuture._1
val
subPage =
Await.result(subPageFuture._2,
Duration.Inf
)
val
href:
String =
{
if
(
path.isEmpty
)
rootPath
+
subPage
._1
+
"/
index.html
"
else
rootPath
+
path.mkString
("","/","/")
+
subPage
._1
+
"/
index.html
"
if
(
path.isEmpty
)
rootPath
+
subPage
Name
+
"/
index.html
"
else
rootPath
+
path.mkString
("","/","/")
+
subPage
Name
+
"/
index.html
"
}
//
buffer.append
("<
li
")
...
...
@@ -30,16 +37,16 @@
// buffer.append(createMenu(subPage._2, path ::: subPage._1 :: Nil, first = false))
// buffer.append("
</li>
")
val listSubmenu = if(subPage.
_2.
subPages.nonEmpty) "dropdown-submenu" else ""
val listSubmenu = if(subPage.subPages.nonEmpty) "dropdown-submenu" else ""
// val subMenuBadgeCount = if(subPage._2.subPages.nonEmpty
&&
first) "
<span
class=
'badge'
>
%d
</span>
".format(subPage._2.subPages.size) else ""
val tabIndex = if (first) " tabindex='-1'" else ""
// val listGroupA = if(subPage._2.subPages.nonEmpty) "list-group-item" else ""
var menuItem: String = "
<li
class=
'%s'
>
".format(listSubmenu) +
"
<a
href=
'%s'
class=
'%s'
%
s
>
".format(href, "", tabIndex) +
"%s".format(subPage
._1
) +
"%s".format(subPage
Name
) +
"
</a>
" +
createMenu(subPage
._2, path ::: subPage._1
:: Nil, first = false) +
createMenu(subPage
, path ::: subPageName
:: Nil, first = false) +
"
</li>
"
buffer.append(menuItem)
...
...
@@ -51,10 +58,10 @@
}
def getSubPage(path:List[String]): ReportPage = {
path.foldLeft(i
ndex
Page)((c, p) => {
path.foldLeft(iPage)((c, p) => {
val foundPages = c.subPages.filter(_._1 == p)
require(foundPages.size == 1, "Name of page not found or duplicate is found, page:: " + p + " in path: " + path)
foundPages.head._2
Await.result(foundPages.head._2, Duration.Inf)
})
}
}#
...
...
@@ -111,9 +118,9 @@
<ul
class=
"nav navbar-nav"
>
<li
class=
"root #if (path.isEmpty) active #end"
>
<a
class=
"navbar-brand"
href=
"${rootPath}index.html"
>
${reportName}
#if (i
ndex
Page.subPages.nonEmpty)
<b
class=
"caret"
></b>
#end
#if (iPage.subPages.nonEmpty)
<b
class=
"caret"
></b>
#end
</a>
${unescape(createMenu(i
ndex
Page))}
${unescape(createMenu(iPage))}
</li>
</ul>
</div>
...
...
@@ -126,9 +133,9 @@
#if(t =
=
0)
<
li
class=
"root"
>
<a
href=
"${rootPath}index.html"
>
Home
#if (i
ndex
Page.subPages.nonEmpty)
<b
class=
"caret"
></b>
#end
#if (iPage.subPages.nonEmpty)
<b
class=
"caret"
></b>
#end
</a>
${unescape(createMenu(i
ndex
Page, Nil, false))}
${unescape(createMenu(iPage, Nil, false))}
</li>
#else
<li
class=
"root #if (t == path.size) active #end"
>
...
...
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilder.scala
View file @
625bb514
...
...
@@ -14,7 +14,7 @@
*/
package
nl.lumc.sasc.biopet.core.report
import
scala.concurrent.
Await
import
scala.concurrent.
{
Await
,
Future
}
import
scala.concurrent.duration.Duration
/**
...
...
@@ -25,46 +25,44 @@ import scala.concurrent.duration.Duration
trait
MultisampleReportBuilder
extends
ReportBuilder
{
/** Method to generate a single sample page */
def
samplePage
(
sampleId
:
Int
,
args
:
Map
[
String
,
Any
])
:
ReportPage
def
samplePage
(
sampleId
:
Int
,
args
:
Map
[
String
,
Any
])
:
Future
[
ReportPage
]
/** Default list of samples, can be override */
def
samplesSections
:
List
[(
String
,
ReportSection
)]
=
{
List
(
(
"Samples"
,
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/samplesList.ssp"
))
)
List
(
"Samples"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/samplesList.ssp"
))
}
/** Method to generate a single library page */
def
libraryPage
(
sampleId
:
Int
,
libraryId
:
Int
,
args
:
Map
[
String
,
Any
])
:
ReportPage
def
libraryPage
(
sampleId
:
Int
,
libraryId
:
Int
,
args
:
Map
[
String
,
Any
])
:
Future
[
ReportPage
]
/** Default list of libraries, can be override */
def
librariesSections
:
List
[(
String
,
ReportSection
)]
=
{
List
(
(
"Libraries"
,
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/librariesList.ssp"
))
)
List
(
"Libraries"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/librariesList.ssp"
))
}
/** Generate the samples page including a single sample page for each sample in the summary */
def
generateSamplesPage
(
args
:
Map
[
String
,
Any
])
:
ReportPage
=
{
val
samples
=
Await
.
result
(
summary
.
getSamples
(
runId
=
Some
(
runId
)),
Duration
.
Inf
)
val
samplePages
=
samples
.
map
(
_
.
id
)
.
map
(
sampleId
=>
sampleId
->
samplePage
(
sampleId
,
args
++
Map
(
"sampleId"
->
Some
(
sampleId
))))
.
toList
ReportPage
(
samplePages
.
map
(
x
=>
samples
.
find
(
_
.
id
==
x
.
_1
).
get
.
name
->
x
.
_2
),
samplesSections
,
args
)
}
def
generateSamplesPage
(
args
:
Map
[
String
,
Any
])
:
Future
[
ReportPage
]
=
summary
.
getSamples
(
runId
=
Some
(
runId
)).
map
{
samples
=>
val
samplePages
=
samples
.
map
(
_
.
id
)
.
map
(
sampleId
=>
sampleId
->
samplePage
(
sampleId
,
args
++
Map
(
"sampleId"
->
Some
(
sampleId
)))
.
map
(
x
=>
x
.
copy
(
subPages
=
x
.
subPages
:::
"Files"
->
filesPage
(
sampleId
,
None
)
::
Nil
)))
.
toList
ReportPage
(
samplePages
.
map
(
x
=>
samples
.
find
(
_
.
id
==
x
.
_1
).
get
.
name
->
x
.
_2
),
samplesSections
,
args
)
}
/** Generate the libraries page for a single sample with a subpage for eacht library */
def
generateLibraryPage
(
args
:
Map
[
String
,
Any
])
:
ReportPage
=
{
def
generateLibraryPage
(
args
:
Map
[
String
,
Any
])
:
Future
[
ReportPage
]
=
{
val
sampleId
=
args
(
"sampleId"
)
match
{
case
Some
(
x
:
Int
)
=>
x
case
None
=>
throw
new
IllegalStateException
(
"Sample not found"
)
}
val
libraries
=
Await
.
result
(
summary
.
getLibraries
(
runId
=
Some
(
runId
),
sampleId
=
Some
(
sampleId
)),
Duration
.
Inf
)
val
libPages
=
libraries
.
map
(
_
.
id
)
.
map
(
libId
=>
libId
->
libraryPage
(
sampleId
,
libId
,
args
++
Map
(
"libId"
->
Some
(
libId
))))
.
toList
ReportPage
(
libPages
.
map
(
x
=>
libraries
.
find
(
_
.
id
==
x
.
_1
).
get
.
name
->
x
.
_2
),
librariesSections
,
args
)
summary
.
getLibraries
(
runId
=
Some
(
runId
),
sampleId
=
Some
(
sampleId
)).
map
{
libraries
=>
val
libPages
=
libraries
.
map
(
_
.
id
)
.
map
(
libId
=>
libId
->
libraryPage
(
sampleId
,
libId
,
args
++
Map
(
"libId"
->
Some
(
libId
)))
.
map
(
x
=>
x
.
copy
(
subPages
=
x
.
subPages
:::
"Files"
->
filesPage
(
sampleId
,
libId
)
::
Nil
)))
.
toList
ReportPage
(
libPages
.
map
(
x
=>
libraries
.
find
(
_
.
id
==
x
.
_1
).
get
.
name
->
x
.
_2
),
librariesSections
,
args
)
}
}
}
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala
View file @
625bb514
...
...
@@ -19,12 +19,13 @@ import java.io._
import
nl.lumc.sasc.biopet.core.ToolCommandFunction
import
nl.lumc.sasc.biopet.utils.summary.db.Schema.
{
Library
,
Module
,
Pipeline
,
Sample
}
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.
{
LibraryId
,
SampleId
}
import
nl.lumc.sasc.biopet.utils.
{
IoUtils
,
Logging
,
ToolCommand
}
import
org.broadinstitute.gatk.utils.commandline.Input
import
org.fusesource.scalate.TemplateEngine
import
scala.collection.mutable
import
scala.concurrent.
{
Await
,
Future
}
import
scala.concurrent.
_
import
scala.concurrent.duration.Duration
import
scala.language.postfixOps
import
scala.language.implicitConversions
...
...
@@ -73,6 +74,7 @@ trait ReportBuilderExtension extends ToolCommandFunction {
trait
ReportBuilder
extends
ToolCommand
{
implicit
lazy
val
ec
=
ReportBuilder
.
ec
implicit
def
toOption
[
T
](
x
:
T
)
:
Option
[
T
]
=
Option
(
x
)
implicit
def
autoWait
[
T
](
x
:
Future
[
T
])
:
T
=
Await
.
result
(
x
,
Duration
.
Inf
)
...
...
@@ -186,35 +188,57 @@ trait ReportBuilder extends ToolCommand {
_setSamples
=
Await
.
result
(
summary
.
getSamples
(
runId
=
Some
(
runId
),
sampleId
=
sampleId
),
Duration
.
Inf
)
_setLibraries
=
Await
.
result
(
summary
.
getLibraries
(
runId
=
Some
(
runId
),
sampleId
=
sampleId
,
libId
=
libId
),
Duration
.
Inf
)
logger
.
info
(
"Copy Base files"
)
val
baseFilesFuture
=
Future
{
logger
.
info
(
"Copy Base files"
)
// Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
val
extOutputDir
:
File
=
new
File
(
cmdArgs
.
outputDir
,
"ext"
)
// Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
val
extOutputDir
:
File
=
new
File
(
cmdArgs
.
outputDir
,
"ext"
)
// Copy each resource files out to the report destination
extFiles
.
par
.
foreach
(
resource
=>
IoUtils
.
copyStreamToFile
(
getClass
.
getResourceAsStream
(
resource
.
resourcePath
),
new
File
(
extOutputDir
,
resource
.
targetPath
),
createDirs
=
true
)
)
// Copy each resource files out to the report destination
extFiles
.
foreach
(
resource
=>
IoUtils
.
copyStreamToFile
(
getClass
.
getResourceAsStream
(
resource
.
resourcePath
),
new
File
(
extOutputDir
,
resource
.
targetPath
),
createDirs
=
true
)
)
}
total
=
ReportBuilder
.
countPages
(
indexPage
)
logger
.
info
(
total
+
" pages to be generated"
)
val
rootPage
=
indexPage
.
map
{
x
=>
x
.
copy
(
subPages
=
x
.
subPages
:::
generalPages
(
sampleId
,
libId
))
}
// total = ReportBuilder.countPages(rootPage)
done
=
0
logger
.
info
(
"Generate pages"
)
val
jobs
=
generatePage
(
summary
,
index
Page
,
cmdArgs
.
outputDir
,
val
jobs
Futures
=
generatePage
(
summary
,
root
Page
,
cmdArgs
.
outputDir
,
args
=
pageArgs
++
cmdArgs
.
pageArgs
.
toMap
++
Map
(
"summary"
->
summary
,
"reportName"
->
reportName
,
"indexPage"
->
indexPage
,
"runId"
->
cmdArgs
.
runId
))
Map
(
"summary"
->
summary
,
"reportName"
->
reportName
,
"indexPage"
->
rootPage
,
"runId"
->
cmdArgs
.
runId
))
total
=
jobsFutures
.
size
logger
.
info
(
total
+
" pages to be generated"
)
logger
.
info
(
jobs
+
" Done"
)
def
wait
(
futures
:
List
[
Future
[
Any
]])
:
Unit
=
{
try
{
Await
.
ready
(
Future
.
sequence
(
futures
),
Duration
.
fromNanos
(
30000000000L
))
}
catch
{
case
e
:
TimeoutException
=>
}
val
notDone
=
futures
.
filter
(!
_
.
isCompleted
)
done
+=
futures
.
size
-
notDone
.
size
if
(
notDone
.
nonEmpty
)
{
logger
.
info
(
s
"$done / $total pages are generated"
)
wait
(
notDone
)
}
}
wait
(
jobsFutures
)
Await
.
ready
(
baseFilesFuture
,
Duration
.
Inf
)
logger
.
info
(
s
"Done, $done pages generated"
)
}
/** This must be implemented, this will be the root page of the report */
def
indexPage
:
ReportPage
def
indexPage
:
Future
[
ReportPage
]
/** This must be implemented, this will become the title of the report */
def
reportName
:
String
...
...
@@ -223,62 +247,112 @@ trait ReportBuilder extends ToolCommand {
* This method will render the page and the subpages recursivly
*
* @param summary The summary object
* @param page Page to render
* @param page
Future
Page to render
* @param outputDir Root output dir of the report
* @param path Path from root to current page
* @param args Args to add to this sub page, are args from current page are passed automaticly
* @return Number of pages including all subpages that are rendered
*/
def
generatePage
(
summary
:
SummaryDb
,
page
:
ReportPage
,
page
Future
:
Future
[
ReportPage
]
,
outputDir
:
File
,
path
:
List
[
String
]
=
Nil
,
args
:
Map
[
String
,
Any
]
=
Map
())
:
Int
=
{
args
:
Map
[
String
,
Any
]
=
Map
())
:
List
[
Future
[
_
]]
=
{
val
pageOutputDir
=
new
File
(
outputDir
,
path
.
mkString
(
File
.
separator
))
pageOutputDir
.
mkdirs
()
val
rootPath
=
"./"
+
Array
.
fill
(
path
.
size
)(
"../"
).
mkString
val
pageArgs
=
args
++
page
.
args
++
Map
(
"page"
->
page
,
"path"
->
path
,
"outputDir"
->
pageOutputDir
,
"rootPath"
->
rootPath
,
"allPipelines"
->
pipelines
,
"allModules"
->
modules
,
"allSamples"
->
samples
,
"allLibraries"
->
libraries
)
// Generating subpages
val
jobs
=
page
.
subPages
.
par
.
flatMap
{
case
(
name
,
subPage
)
=>
Some
(
generatePage
(
summary
,
subPage
,
outputDir
,
path
:::
name
::
Nil
,
pageArgs
))
case
_
=>
None
def
pageArgs
(
page
:
ReportPage
)
=
{
val
rootPath
=
"./"
+
Array
.
fill
(
path
.
size
)(
"../"
).
mkString
args
++
page
.
args
++
Map
(
"page"
->
page
,
"path"
->
path
,
"outputDir"
->
pageOutputDir
,
"rootPath"
->
rootPath
,
"allPipelines"
->
pipelines
,
"allModules"
->
modules
,
"allSamples"
->
samples
,
"allLibraries"
->
libraries
)
}
val
output
=
ReportBuilder
.
renderTemplate
(
"/nl/lumc/sasc/biopet/core/report/main.ssp"
,
pageArgs
++
Map
(
"args"
->
pageArgs
))
val
subPageJobs
=
pageFuture
.
map
{
page
=>
// Generating subpages
page
.
subPages
.
flatMap
{
case
(
name
,
subPage
)
=>
generatePage
(
summary
,
subPage
,
outputDir
,
path
:::
name
::
Nil
,
pageArgs
(
page
))
}
}
val
file
=
new
File
(
pageOutputDir
,
"index.html"
)
val
writer
=
new
PrintWriter
(
file
)
writer
.
println
(
output
)
writer
.
close
()
val
renderFuture
=
pageFuture
.
map
{
page
=>
pageOutputDir
.
mkdirs
()
done
+=
1
if
(
done
%
100
==
0
)
logger
.
info
(
done
+
" Done, "
+
(
done
.
toDouble
/
total
*
100
)
+
"%"
)
jobs
.
sum
+
1
val
file
=
new
File
(
pageOutputDir
,
"index.html"
)
logger
.
info
(
s
"Start rendering: $file"
)
val
output
=
ReportBuilder
.
renderTemplate
(
"/nl/lumc/sasc/biopet/core/report/main.ssp"
,
pageArgs
(
page
)
++
Map
(
"args"
->
pageArgs
(
page
)))
val
writer
=
new
PrintWriter
(
file
)
writer
.
println
(
output
)
writer
.
close
()
logger
.
info
(
s
"Done rendering: $file"
)
}
renderFuture
::
Await
.
result
(
subPageJobs
,
Duration
.
Inf
)
}
def
pipelineName
:
String
/** Files page, can be used general or at sample level */
def
filesPage
(
sampleId
:
Option
[
Int
]
=
None
,
libraryId
:
Option
[
Int
]
=
None
)
:
Future
[
ReportPage
]
=
{
val
dbFiles
=
summary
.
getFiles
(
runId
,
sample
=
sampleId
.
map
(
SampleId
),
library
=
libraryId
.
map
(
LibraryId
))
.
map
(
_
.
groupBy
(
_
.
pipelineId
))
val
modulePages
=
dbFiles
.
map
(
_
.
map
{
case
(
pipelineId
,
files
)
=>
val
moduleSections
=
files
.
groupBy
(
_
.
moduleId
).
map
{
case
(
moduleId
,
files
)
=>
val
moduleName
:
Future
[
String
]
=
moduleId
match
{
case
Some
(
id
)
=>
summary
.
getModuleName
(
pipelineId
,
id
).
map
(
_
.
getOrElse
(
"Pipeline"
))
case
_
=>
Future
.
successful
(
"Pipeline"
)
}
moduleName
.
map
(
_
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/files.ssp"
,
Map
(
"files"
->
files
)))
}
val
moduleSectionsSorted
=
moduleSections
.
find
(
_
.
_1
==
"Pipeline"
)
++
moduleSections
.
filter
(
_
.
_1
!=
"Pipeline"
)
summary
.
getPipelineName
(
pipelineId
=
pipelineId
)
.
map
(
_
.
get
->
Future
.
sequence
(
moduleSectionsSorted
)
.
map
(
sections
=>
ReportPage
(
Nil
,
sections
.
toList
,
Map
())))
})
val
pipelineFiles
=
summary
.
getPipelineId
(
runId
,
pipelineName
)
.
flatMap
(
pipelinelineId
=>
dbFiles
.
map
(
x
=>
x
.
get
(
pipelinelineId
.
get
).
getOrElse
(
Seq
())
.
filter
(
_
.
moduleId
.
isEmpty
)))
modulePages
.
flatMap
(
Future
.
sequence
(
_
)).
map
(
x
=>
ReportPage
(
x
.
toList
,
s
"$pipelineName files"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/files.ssp"
,
Map
(
"files"
->
Await
.
result
(
pipelineFiles
,
Duration
.
Inf
)))
::
"Sub pipelines/modules"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/fileModules.ssp"
,
Map
(
"pipelineIds"
->
Await
.
result
(
dbFiles
.
map
(
_
.
keys
.
toList
),
Duration
.
Inf
)))
::
Nil
,
Map
()))
}
/** This generate general pages that all reports should have */
def
generalPages
(
sampleId
:
Option
[
Int
],
libId
:
Option
[
Int
])
:
List
[(
String
,
Future
[
ReportPage
])]
=
List
(
"Versions"
->
Future
.
successful
(
ReportPage
(
List
(),
List
(
"Executables"
->
ReportSection
(
"/nl/lumc/sasc/biopet/core/report/executables.ssp"
)),
Map
())),
"Files"
->
filesPage
(
sampleId
,
libId
)
)
}
object
ReportBuilder
{
implicit
lazy
val
ec
=
ExecutionContext
.
global
/** Single template render engine, this will have a cache for all compile templates */
protected
val
engine
=
new
TemplateEngine
()
engine
.
allowReload
=
false
/** This will give the total number of pages including all nested pages */
def
countPages
(
page
:
ReportPage
)
:
Int
=
{
page
.
subPages
.
map
(
x
=>
countPages
(
x
.
_2
)).
fold
(
1
)(
_
+
_
)
}
//
def countPages(page: ReportPage): Int = {
//
page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
//
}
/**
* This method will render a template that is located in the classpath / jar
...
...
@@ -287,7 +361,7 @@ object ReportBuilder {
* @return Rendered result of template
*/
def
renderTemplate
(
location
:
String
,
args
:
Map
[
String
,
Any
]
=
Map
())
:
String
=
{
Logging
.
logger
.
info
(
"Rendering: "
+
location
)
Logging
.
logger
.
debug
(
"Rendering: "
+
location
)
engine
.
layout
(
location
,
args
)
}
...
...
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportPage.scala
View file @
625bb514
...
...
@@ -14,6 +14,8 @@
*/
package
nl.lumc.sasc.biopet.core.report
import
scala.concurrent.Future
/**
* Created by pjvan_thof on 3/27/15.
*
...
...
@@ -21,6 +23,6 @@ package nl.lumc.sasc.biopet.core.report
* @param sections Sections for this page
* @param args Arguments for this page, this arguments get passed to all section and subpages
*/
case
class
ReportPage
(
subPages
:
List
[(
String
,
ReportPage
)],
case
class
ReportPage
(
subPages
:
List
[(
String
,
Future
[
ReportPage
]
)],
sections
:
List
[(
String
,
ReportSection
)],
args
:
Map
[
String
,
Any
])
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala
View file @
625bb514
...
...
@@ -26,6 +26,7 @@ import nl.lumc.sasc.biopet.LastCommitHash
import
scala.collection.mutable
import
scala.concurrent.Await
import
scala.concurrent.duration.Duration
import
scala.concurrent.ExecutionContext.Implicits.global
import
scala.io.Source
/**
...
...
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/MultisampleReportBuilderTest.scala
View file @
625bb514
...
...
@@ -16,6 +16,7 @@ package nl.lumc.sasc.biopet.core.report
import
java.io.File
import
java.nio.file.Paths
import
java.sql.Date
import
com.google.common.io.Files
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
...
...
@@ -23,7 +24,7 @@ import org.scalatest.Matchers
import
org.scalatest.testng.TestNGSuite
import
org.testng.annotations.Test
import
scala.concurrent.
Await
import
scala.concurrent.
{
Await
,
Future
}
import
scala.concurrent.duration.Duration
/**
...
...
@@ -37,19 +38,26 @@ class MultisampleReportBuilderTest extends TestNGSuite with Matchers {
@Test
def
testGeneratePages
()
:
Unit
=
{
val
builder
=
new
MultisampleReportBuilder
{
def
pipelineName
=
"test"
def
reportName
:
String
=
"test"
def
indexPage
:
ReportPage
=
ReportPage
(
"Samples"
->
generateSamplesPage
(
Map
())
::
Nil
,
Nil
,
Map
(
))
def
indexPage
:
Future
[
ReportPage
]
=
Future
(
ReportPage
(
"Samples"
->
generateSamplesPage
(
Map
())
::
Nil
,
Nil
,
Map
()
))
def
samplePage
(
sampleId
:
Int
,
args
:
Map
[
String
,
Any
])
:
ReportPage
=
ReportPage
(
"Libraries"
->
generateLibraryPage
(
Map
(
"sampleId"
->
Some
(
sampleId
)))
::
Nil
,
Nil
,
Map
(
))
def
samplePage
(
sampleId
:
Int
,
args
:
Map
[
String
,
Any
])
:
Future
[
ReportPage
]
=
Future
(
ReportPage
(
"Libraries"
->
generateLibraryPage
(
Map
(
"sampleId"
->
Some
(
sampleId
)))
::
Nil
,
Nil
,
Map
()
))
def
libraryPage
(
sampleId
:
Int
,
libraryId
:
Int
,
args
:
Map
[
String
,
Any
])
=
ReportPage
(
Nil
,
Nil
,
Map
(
))
def
libraryPage
(
sampleId
:
Int
,
libraryId
:
Int
,
args
:
Map
[
String
,
Any
])
=
Future
(
ReportPage
(
Nil
,
Nil
,
Map
()
))
}
import
scala.concurrent.ExecutionContext.Implicits.global
val
dbFile
=
File
.
createTempFile
(
"summary."
,
".db"
)
dbFile
.
deleteOnExit
()
val
db
=
SummaryDb
.
openSqliteSummary
(
dbFile
)
db
.
createTables
()
Await
.
ready
(
db
.
createPipeline
(
"test"
,
0
),
Duration
.
Inf
)
Await
.
ready
(
db
.
createRun
(
"test"
,
""
,
""
,
""
,
new
Date
(
System
.
currentTimeMillis
())),
Duration
.
Inf
)
val
sample
=
Some
(
"sampleName"
)
val
lib
=
Some
(
"libName"
)
...
...
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala
View file @
625bb514
...
...
@@ -16,6 +16,7 @@ package nl.lumc.sasc.biopet.core.report
import
java.io.File
import
java.nio.file.Paths
import
java.sql.Date
import
com.google.common.io.Files
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb