ReportBuilder.scala 10.1 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample, Run }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import org.broadinstitute.gatk.utils.commandline.Input
24
import org.fusesource.scalate.TemplateEngine
25

Peter van 't Hof's avatar
Peter van 't Hof committed
26
import scala.collection.mutable
Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.concurrent.{ Await, Future }
28
import scala.concurrent.duration.Duration
29
import scala.language.postfixOps
Peter van 't Hof's avatar
Peter van 't Hof committed
30
import scala.language.implicitConversions
31
32

/**
33
34
35
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
36
 */
37
trait ReportBuilderExtension extends ToolCommandFunction {
38

39
  /** Report builder object */
40
  def builder: ReportBuilder
41

Peter van 't Hof's avatar
Peter van 't Hof committed
42
  def toolObject: ReportBuilder = builder
Peter van 't Hof's avatar
Peter van 't Hof committed
43

44
  @Input(required = true)
45
46
47
  var summaryDbFile: File = _

  var runId: Option[Int] = None
48

49
  /** OutputDir for the report  */
50
51
  var outputDir: File = _

52
  /** Arguments that are passed on the commandline */
53
54
  var args: Map[String, String] = Map()

55
56
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
57

58
59
  override def beforeGraph(): Unit = {
    super.beforeGraph()
60
61
62
63
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

64
  /** Command to generate the report */
65
66
  override def cmdLine: String = {
    super.cmdLine +
67
68
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
69
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
70
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
71
72
73
  }
}

74
75
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
Peter van 't Hof committed
76
77
  implicit def toOption[T](x: T): Option[T] = Option(x)
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
78

79
  case class Args(summaryDbFile: File = null,
80
                  outputDir: File = null,
81
                  runId: Int = 0,
82
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
83
84

  class OptParser extends AbstractOptParser {
85
86
87
88
89
90
91

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

92
93
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
94
95
96
97
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
98
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
99
      c.copy(outputDir = x)
100
101
    } text "Output HTML report files to this directory"

102
103
104
105
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
106
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
107
108
      c.copy(pageArgs = c.pageArgs ++ x)
    }
109
110
  }

111
  /** summary object internaly */
112
  private var setSummary: SummaryDb = _
113

114
  /** Retrival of summary, read only */
Peter van 't Hof's avatar
Peter van 't Hof committed
115
  final def summary: SummaryDb = setSummary
116

117
118
  private var setRunId: Int = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
119
120
121
122
123
  final def runId: Int = setRunId

  private var _setRun: Run = _

  final def run: Run = _setRun
124

Peter van 't Hof's avatar
Peter van 't Hof committed
125
  private var _setPipelines = Seq[Pipeline]()
Peter van 't Hof's avatar
Peter van 't Hof committed
126
  final def pipelines: Seq[Pipeline] = _setPipelines
Peter van 't Hof's avatar
Peter van 't Hof committed
127
  private var _setModules = Seq[Module]()
Peter van 't Hof's avatar
Peter van 't Hof committed
128
  final def modules: Seq[Module] = _setModules
Peter van 't Hof's avatar
Peter van 't Hof committed
129
  private var _setSamples = Seq[Sample]()
Peter van 't Hof's avatar
Peter van 't Hof committed
130
  final def samples: Seq[Sample] = _setSamples
Peter van 't Hof's avatar
Peter van 't Hof committed
131
  private var _setLibraries = Seq[Library]()
Peter van 't Hof's avatar
Peter van 't Hof committed
132
  final def libraries: Seq[Library] = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
133

134
  /** default args that are passed to all page withing the report */
135
136
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
137
138
139
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
140
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
141
  protected[report] def sampleId: Option[Int] = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
142
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
143
  protected[report] def libId: Option[Int] = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
144

145
146
  case class ExtFile(resourcePath: String, targetPath: String)

Peter van 't Hof's avatar
Peter van 't Hof committed
147
  def extFiles: List[ExtFile] = List(
148
149
150
151
152
153
154
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
155
    "js/d3.v3.5.5.min.js",
156
157
158
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
159
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
160

161
  /** Main function to for building the report */
162
163
164
165
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
166
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
167
168
169
170

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

171
    setSummary = SummaryDb.openReadOnlySqliteSummary(cmdArgs.summaryDbFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
172
173
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
174
    cmdArgs.pageArgs.get("sampleId") match {
175
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
176
177
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
178
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
179
180
181
    }

    cmdArgs.pageArgs.get("libId") match {
182
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
183
184
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
185
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
186
187
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
188
    _setRun = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
Peter van 't Hof's avatar
Peter van 't Hof committed
189
190
    _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
    _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
191
192
193
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

194
195
    logger.info("Copy Base files")

Peter van 't Hof's avatar
Peter van 't Hof committed
196
    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
197
198
    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")

199
200
201
202
203
204
205
206
    // Copy each resource files out to the report destination
    extFiles.par.foreach(
      resource =>
        IoUtils.copyStreamToFile(
          getClass.getResourceAsStream(resource.resourcePath),
          new File(extOutputDir, resource.targetPath),
          createDirs = true)
    )
207

208
    total = ReportBuilder.countPages(indexPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
209
210
    logger.info(total + " pages to be generated")

Peter van 't Hof's avatar
Peter van 't Hof committed
211
212
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
213
    logger.info("Generate pages")
Peter van 't Hof's avatar
Peter van 't Hof committed
214
    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
215
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
216
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
217

Peter van 't Hof's avatar
Peter van 't Hof committed
218
    logger.info(jobs + " Done")
219
220
  }

221
  /** This must be implemented, this will be the root page of the report */
222
223
  def indexPage: ReportPage

224
  /** This must be implemented, this will become the title of the report */
225
226
  def reportName: String

227
228
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
229
   *
230
231
232
233
234
235
236
   * @param summary The summary object
   * @param page Page to render
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
237
  def generatePage(summary: SummaryDb,
238
239
240
                   page: ReportPage,
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
Peter van 't Hof committed
241
                   args: Map[String, Any] = Map()): Int = {
242

Peter van 't Hof's avatar
Peter van 't Hof committed
243
244
    val pageOutputDir = new File(outputDir, path.mkString(File.separator))
    pageOutputDir.mkdirs()
245
    val rootPath = "./" + Array.fill(path.size)("../").mkString
246
247
    val pageArgs = args ++ page.args ++
      Map("page" -> page,
Peter van 't Hof's avatar
Peter van 't Hof committed
248
        "run" -> run,
249
250
        "path" -> path,
        "outputDir" -> pageOutputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
251
252
253
254
255
        "rootPath" -> rootPath,
        "allPipelines" -> pipelines,
        "allModules" -> modules,
        "allSamples" -> samples,
        "allLibraries" -> libraries
256
      )
257

258
    // Generating subpages
259
260
261
    val jobs = page.subPages.par.flatMap {
      case (name, subPage) => Some(generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs))
      case _               => None
262
263
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
264
265
266
267
    val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
      pageArgs ++ Map("args" -> pageArgs))

    val file = new File(pageOutputDir, "index.html")
268
269
270
271
    val writer = new PrintWriter(file)
    writer.println(output)
    writer.close()

Peter van 't Hof's avatar
Peter van 't Hof committed
272
273
    done += 1
    if (done % 100 == 0) logger.info(done + " Done, " + (done.toDouble / total * 100) + "%")
274
    jobs.sum + 1
275
276
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
277
278
279

object ReportBuilder {

280
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
281
  protected val engine = new TemplateEngine()
282
  engine.allowReload = false
283

284
285
286
287
288
  /** This will give the total number of pages including all nested pages */
  def countPages(page: ReportPage): Int = {
    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
289
290
291
292
293
294
295
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
296
297
    Logging.logger.info("Rendering: " + location)

298
    engine.layout(location, args)
299
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
300
}