ReportBuilder.scala 13.4 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample, Run }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ LibraryId, SampleId }
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.broadinstitute.gatk.utils.commandline.Input
25
import org.fusesource.scalate.TemplateEngine
26

Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.collection.mutable
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
28
import scala.concurrent._
29
import scala.concurrent.duration.Duration
30
import scala.language.postfixOps
Peter van 't Hof's avatar
Peter van 't Hof committed
31
import scala.language.implicitConversions
32
33

/**
34
35
36
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
37
 */
38
trait ReportBuilderExtension extends ToolCommandFunction {
39

40
  /** Report builder object */
41
  def builder: ReportBuilder
42

Peter van 't Hof's avatar
Peter van 't Hof committed
43
  def toolObject: ReportBuilder = builder
Peter van 't Hof's avatar
Peter van 't Hof committed
44

45
  @Input(required = true)
46
47
48
  var summaryDbFile: File = _

  var runId: Option[Int] = None
49

50
  /** OutputDir for the report  */
51
52
  var outputDir: File = _

53
  /** Arguments that are passed on the commandline */
54
55
  var args: Map[String, String] = Map()

56
57
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
58

59
60
  override def beforeGraph(): Unit = {
    super.beforeGraph()
61
62
63
64
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

65
  /** Command to generate the report */
66
67
  override def cmdLine: String = {
    super.cmdLine +
68
69
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
70
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
71
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
72
73
74
  }
}

75
76
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
77
  implicit lazy val ec = ReportBuilder.ec
Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
  implicit def toOption[T](x: T): Option[T] = Option(x)
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
80

81
  case class Args(summaryDbFile: File = null,
82
                  outputDir: File = null,
83
                  runId: Int = 0,
84
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
85
86

  class OptParser extends AbstractOptParser {
87
88
89
90
91
92
93

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

94
95
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
96
97
98
99
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
100
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
101
      c.copy(outputDir = x)
102
103
    } text "Output HTML report files to this directory"

104
105
106
107
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
108
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
109
110
      c.copy(pageArgs = c.pageArgs ++ x)
    }
111
112
  }

113
  /** summary object internaly */
114
  private var setSummary: SummaryDb = _
115

116
  /** Retrival of summary, read only */
Peter van 't Hof's avatar
Peter van 't Hof committed
117
  final def summary: SummaryDb = setSummary
118

119
120
  private var setRunId: Int = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
121
122
123
124
125
  final def runId: Int = setRunId

  private var _setRun: Run = _

  final def run: Run = _setRun
126

Peter van 't Hof's avatar
Peter van 't Hof committed
127
  private var _setPipelines = Seq[Pipeline]()
Peter van 't Hof's avatar
Peter van 't Hof committed
128
  final def pipelines: Seq[Pipeline] = _setPipelines
Peter van 't Hof's avatar
Peter van 't Hof committed
129
  private var _setModules = Seq[Module]()
Peter van 't Hof's avatar
Peter van 't Hof committed
130
  final def modules: Seq[Module] = _setModules
Peter van 't Hof's avatar
Peter van 't Hof committed
131
  private var _setSamples = Seq[Sample]()
Peter van 't Hof's avatar
Peter van 't Hof committed
132
  final def samples: Seq[Sample] = _setSamples
Peter van 't Hof's avatar
Peter van 't Hof committed
133
  private var _setLibraries = Seq[Library]()
Peter van 't Hof's avatar
Peter van 't Hof committed
134
  final def libraries: Seq[Library] = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
135

136
  /** default args that are passed to all page withing the report */
137
138
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
139
140
141
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
142
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
143
  protected[report] def sampleId: Option[Int] = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
144
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
145
  protected[report] def libId: Option[Int] = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
146

147
148
  case class ExtFile(resourcePath: String, targetPath: String)

Peter van 't Hof's avatar
Peter van 't Hof committed
149
  def extFiles: List[ExtFile] = List(
150
151
152
153
154
155
156
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
157
    "js/d3.v3.5.5.min.js",
158
159
160
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
161
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
162

163
  /** Main function to for building the report */
164
165
166
167
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
168
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
169
170
171
172

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

173
    setSummary = SummaryDb.openReadOnlySqliteSummary(cmdArgs.summaryDbFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
174
175
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
176
    cmdArgs.pageArgs.get("sampleId") match {
177
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
178
179
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
180
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
181
182
183
    }

    cmdArgs.pageArgs.get("libId") match {
184
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
185
186
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
187
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
188
189
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
190
    _setRun = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
Peter van 't Hof's avatar
Peter van 't Hof committed
191
192
    _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
    _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
193
194
195
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
196
197
    val baseFilesFuture = Future {
      logger.info("Copy Base files")
198

Peter van 't Hof's avatar
Peter van 't Hof committed
199
200
      // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
      val extOutputDir: File = new File(cmdArgs.outputDir, "ext")
201

Peter van 't Hof's avatar
Peter van 't Hof committed
202
203
204
205
206
207
208
209
210
      // Copy each resource files out to the report destination
      extFiles.foreach(
        resource =>
          IoUtils.copyStreamToFile(
            getClass.getResourceAsStream(resource.resourcePath),
            new File(extOutputDir, resource.targetPath),
            createDirs = true)
      )
    }
211

Peter van 't Hof's avatar
Peter van 't Hof committed
212
    val rootPage = indexPage.map { x => x.copy(subPages = x.subPages ::: generalPages(sampleId, libId)) }
Peter van 't Hof's avatar
Peter van 't Hof committed
213

Peter van 't Hof's avatar
Peter van 't Hof committed
214
    //    total = ReportBuilder.countPages(rootPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
215
216
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
217
    logger.info("Generate pages")
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
218
    val jobsFutures = generatePage(summary, rootPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
219
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
Peter van 't Hof's avatar
Peter van 't Hof committed
220
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
221

Peter van 't Hof's avatar
Peter van 't Hof committed
222
223
    total = jobsFutures.size
    logger.info(total + " pages to be generated")
224

Peter van 't Hof's avatar
Peter van 't Hof committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    def wait(futures: List[Future[Any]]): Unit = {
      try {
        Await.ready(Future.sequence(futures), Duration.fromNanos(30000000000L))
      } catch {
        case e: TimeoutException =>
      }
      val notDone = futures.filter(!_.isCompleted)
      done += futures.size - notDone.size
      if (notDone.nonEmpty) {
        logger.info(s"$done / $total pages are generated")
        wait(notDone)
      }
    }

    wait(jobsFutures)
    Await.ready(baseFilesFuture, Duration.Inf)

    logger.info(s"Done, $done pages generated")
243
244
  }

245
  /** This must be implemented, this will be the root page of the report */
Peter van 't Hof's avatar
Peter van 't Hof committed
246
  def indexPage: Future[ReportPage]
247

248
  /** This must be implemented, this will become the title of the report */
249
250
  def reportName: String

251
252
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
253
   *
254
   * @param summary The summary object
Peter van 't Hof's avatar
Peter van 't Hof committed
255
   * @param pageFuture Page to render
256
257
258
259
260
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
261
  def generatePage(summary: SummaryDb,
Peter van 't Hof's avatar
Peter van 't Hof committed
262
                   pageFuture: Future[ReportPage],
263
264
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
265
                   args: Map[String, Any] = Map()): List[Future[_]] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
266
    val pageOutputDir = new File(outputDir, path.mkString(File.separator))
267

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
268
    def pageArgs(page: ReportPage) = {
Peter van 't Hof's avatar
Peter van 't Hof committed
269
      val rootPath = "./" + Array.fill(path.size)("../").mkString
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
270
      args ++ page.args ++
Peter van 't Hof's avatar
Peter van 't Hof committed
271
        Map("page" -> page,
272
          "run" -> run,
Peter van 't Hof's avatar
Peter van 't Hof committed
273
274
275
276
277
278
279
280
          "path" -> path,
          "outputDir" -> pageOutputDir,
          "rootPath" -> rootPath,
          "allPipelines" -> pipelines,
          "allModules" -> modules,
          "allSamples" -> samples,
          "allLibraries" -> libraries
        )
281
282
    }

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
283
    val subPageJobs = pageFuture.map { page =>
Peter van 't Hof's avatar
Peter van 't Hof committed
284
      // Generating subpages
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
285
286
      page.subPages.flatMap {
        case (name, subPage) => generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs(page))
Peter van 't Hof's avatar
Peter van 't Hof committed
287
      }
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
288
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
289

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
290
    val renderFuture = pageFuture.map { page =>
Peter van 't Hof's avatar
Peter van 't Hof committed
291
      pageOutputDir.mkdirs()
292

Peter van 't Hof's avatar
Peter van 't Hof committed
293
294
      val file = new File(pageOutputDir, "index.html")
      logger.info(s"Start rendering: $file")
Peter van 't Hof's avatar
Peter van 't Hof committed
295

Peter van 't Hof's avatar
Peter van 't Hof committed
296
297
      val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
        pageArgs(page) ++ Map("args" -> pageArgs(page)))
Peter van 't Hof's avatar
Peter van 't Hof committed
298

Peter van 't Hof's avatar
Peter van 't Hof committed
299
300
301
302
      val writer = new PrintWriter(file)
      writer.println(output)
      writer.close()
      logger.info(s"Done rendering: $file")
Peter van 't Hof's avatar
Peter van 't Hof committed
303

304
305
    }

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
306
    renderFuture :: Await.result(subPageJobs, Duration.Inf)
307
  }
308

Peter van 't Hof's avatar
Peter van 't Hof committed
309
  def pipelineName: String
310
311
312
313
314
315
316
317
318
319
320
321

  /** Files page, can be used general or at sample level */
  def filesPage(sampleId: Option[Int] = None, libraryId: Option[Int] = None): Future[ReportPage] = {
    val dbFiles = summary.getFiles(runId, sample = sampleId.map(SampleId),
      library = libraryId.map(LibraryId))
      .map(_.groupBy(_.pipelineId))
    val modulePages = dbFiles.map(_.map {
      case (pipelineId, files) =>
        val moduleSections = files.groupBy(_.moduleId).map {
          case (moduleId, files) =>
            val moduleName: Future[String] = moduleId match {
              case Some(id) => summary.getModuleName(pipelineId, id).map(_.getOrElse("Pipeline"))
Peter van 't Hof's avatar
Peter van 't Hof committed
322
              case _        => Future.successful("Pipeline")
323
324
325
326
            }
            moduleName.map(_ -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> files)))
        }
        val moduleSectionsSorted = moduleSections.find(_._1 == "Pipeline") ++ moduleSections.filter(_._1 != "Pipeline")
Peter van 't Hof's avatar
Peter van 't Hof committed
327
328
329
        summary.getPipelineName(pipelineId = pipelineId)
          .map(_.get -> Future.sequence(moduleSectionsSorted)
            .map(sections => ReportPage(Nil, sections.toList, Map())))
330
331
    })

Peter van 't Hof's avatar
Peter van 't Hof committed
332
333
334
335
    val pipelineFiles = summary.getPipelineId(runId, pipelineName)
      .flatMap(pipelinelineId => dbFiles
        .map(x => x.get(pipelinelineId.get).getOrElse(Seq())
          .filter(_.moduleId.isEmpty)))
336
337
338
339
340
341
342
343

    modulePages.flatMap(Future.sequence(_)).map(x => ReportPage(x.toList,
      s"$pipelineName files" -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> Await.result(pipelineFiles, Duration.Inf))) ::
        "Sub pipelines/modules" -> ReportSection("/nl/lumc/sasc/biopet/core/report/fileModules.ssp", Map("pipelineIds" -> Await.result(dbFiles.map(_.keys.toList), Duration.Inf))) :: Nil, Map()))
  }

  /** This generate general pages that all reports should have */
  def generalPages(sampleId: Option[Int], libId: Option[Int]): List[(String, Future[ReportPage])] = List(
Peter van 't Hof's avatar
Peter van 't Hof committed
344
    "Versions" -> Future.successful(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp")), Map())),
345
346
347
    "Files" -> filesPage(sampleId, libId)
  )

348
}
Peter van 't Hof's avatar
Peter van 't Hof committed
349
350
351

object ReportBuilder {

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
352
353
  implicit lazy val ec = ExecutionContext.global

354
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
355
  protected val engine = new TemplateEngine()
356
  engine.allowReload = false
357

358
  /** This will give the total number of pages including all nested pages */
Peter van 't Hof's avatar
Peter van 't Hof committed
359
360
361
  //  def countPages(page: ReportPage): Int = {
  //    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  //  }
362

Peter van 't Hof's avatar
Peter van 't Hof committed
363
364
365
366
367
368
369
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
370
    Logging.logger.debug("Rendering: " + location)
Peter van 't Hof's avatar
Peter van 't Hof committed
371

372
    engine.layout(location, args)
373
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
374
}