ReportBuilder.scala 13 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
23
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ LibraryId, SampleId }
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.broadinstitute.gatk.utils.commandline.Input
25
import org.fusesource.scalate.TemplateEngine
26

Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.collection.mutable
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
28
import scala.concurrent._
29
import scala.concurrent.duration.Duration
30
import scala.language.postfixOps
Peter van 't Hof's avatar
Peter van 't Hof committed
31
import scala.language.implicitConversions
32
33

/**
34
35
36
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
37
 */
38
trait ReportBuilderExtension extends ToolCommandFunction {
39

40
  /** Report builder object */
41
  def builder: ReportBuilder
42

Peter van 't Hof's avatar
Peter van 't Hof committed
43
44
  def toolObject = builder

45
  @Input(required = true)
46
47
48
  var summaryDbFile: File = _

  var runId: Option[Int] = None
49

50
  /** OutputDir for the report  */
51
52
  var outputDir: File = _

53
  /** Arguments that are passed on the commandline */
54
55
  var args: Map[String, String] = Map()

56
57
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
58

59
60
  override def beforeGraph(): Unit = {
    super.beforeGraph()
61
62
63
64
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

65
  /** Command to generate the report */
66
67
  override def cmdLine: String = {
    super.cmdLine +
68
69
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
70
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
71
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
72
73
74
  }
}

75
76
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
77
  implicit lazy val ec = ReportBuilder.ec
Peter van 't Hof's avatar
Peter van 't Hof committed
78
  implicit def toOption[T](x: T): Option[T] = Option(x)
Peter van 't Hof's avatar
Peter van 't Hof committed
79
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
80

81
  case class Args(summaryDbFile: File = null,
82
                  outputDir: File = null,
83
                  runId: Int = 0,
84
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
85
86

  class OptParser extends AbstractOptParser {
87
88
89
90
91
92
93

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

94
95
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
96
97
98
99
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
100
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
101
      c.copy(outputDir = x)
102
103
    } text "Output HTML report files to this directory"

104
105
106
107
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
108
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
109
110
      c.copy(pageArgs = c.pageArgs ++ x)
    }
111
112
  }

113
  /** summary object internaly */
114
  private var setSummary: SummaryDb = _
115

116
  /** Retrival of summary, read only */
117
118
  final def summary = setSummary

119
120
121
122
  private var setRunId: Int = 0

  final def runId = setRunId

Peter van 't Hof's avatar
Peter van 't Hof committed
123
124
125
126
  private var _setPipelines = Seq[Pipeline]()
  final def pipelines = _setPipelines
  private var _setModules = Seq[Module]()
  final def modules = _setModules
Peter van 't Hof's avatar
Peter van 't Hof committed
127
128
129
130
  private var _setSamples = Seq[Sample]()
  final def samples = _setSamples
  private var _setLibraries = Seq[Library]()
  final def libraries = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
131

132
  /** default args that are passed to all page withing the report */
133
134
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
135
136
137
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
138
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
139
  protected[report] def sampleId = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
140
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
141
  protected[report] def libId = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
142

143
144
  case class ExtFile(resourcePath: String, targetPath: String)

145
146
147
148
149
150
151
152
  def extFiles = List(
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
153
    "js/d3.v3.5.5.min.js",
154
155
156
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
157
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
158

159
  /** Main function to for building the report */
160
161
162
163
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
164
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
165
166
167
168

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

169
    setSummary = SummaryDb.openReadOnlySqliteSummary(cmdArgs.summaryDbFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
170
171
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
172
    cmdArgs.pageArgs.get("sampleId") match {
173
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
174
175
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
176
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
177
178
179
    }

    cmdArgs.pageArgs.get("libId") match {
180
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
181
182
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
183
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
184
185
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
186
187
    _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
    _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
188
189
190
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    val baseFilesFuture = Future {
      logger.info("Copy Base files")

      // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
      val extOutputDir: File = new File(cmdArgs.outputDir, "ext")

      // Copy each resource files out to the report destination
      extFiles.foreach(
        resource =>
          IoUtils.copyStreamToFile(
            getClass.getResourceAsStream(resource.resourcePath),
            new File(extOutputDir, resource.targetPath),
            createDirs = true)
      )
    }
206

Peter van 't Hof's avatar
Peter van 't Hof committed
207
    val rootPage = indexPage.map { x => x.copy(subPages = x.subPages ::: generalPages(sampleId, libId)) }
Peter van 't Hof's avatar
Peter van 't Hof committed
208
209

    //    total = ReportBuilder.countPages(rootPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
210
211
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
212
    logger.info("Generate pages")
Peter van 't Hof's avatar
Peter van 't Hof committed
213
    val jobs = generatePage(summary, rootPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
214
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
Peter van 't Hof's avatar
Peter van 't Hof committed
215
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
216

Peter van 't Hof's avatar
Peter van 't Hof committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
    val jobsFutures = Await.result(jobs, Duration.Inf)

    total = jobsFutures.size
    logger.info(total + " pages to be generated")

    def wait(futures: List[Future[Any]]): Unit = {
      try {
        Await.ready(Future.sequence(futures), Duration.fromNanos(30000000000L))
      } catch {
        case e: TimeoutException =>
      }
      val notDone = futures.filter(!_.isCompleted)
      done += futures.size - notDone.size
      if (notDone.nonEmpty) {
        logger.info(s"$done / $total pages are generated")
        wait(notDone)
      }
    }

    wait(jobsFutures)
    Await.ready(baseFilesFuture, Duration.Inf)

    logger.info(s"Done, $done pages generated")
240
241
  }

242
  /** This must be implemented, this will be the root page of the report */
Peter van 't Hof's avatar
Peter van 't Hof committed
243
  def indexPage: Future[ReportPage]
244

245
  /** This must be implemented, this will become the title of the report */
246
247
  def reportName: String

248
249
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
250
   *
251
   * @param summary The summary object
Peter van 't Hof's avatar
Peter van 't Hof committed
252
   * @param pageFuture Page to render
253
254
255
256
257
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
258
  def generatePage(summary: SummaryDb,
Peter van 't Hof's avatar
Peter van 't Hof committed
259
                   pageFuture: Future[ReportPage],
260
261
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
Peter van 't Hof committed
262
263
                   args: Map[String, Any] = Map()): Future[List[Future[_]]] = {
    pageFuture.map { page =>
Peter van 't Hof's avatar
Peter van 't Hof committed
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
      val pageOutputDir = new File(outputDir, path.mkString(File.separator))
      pageOutputDir.mkdirs()
      val rootPath = "./" + Array.fill(path.size)("../").mkString
      val pageArgs = args ++ page.args ++
        Map("page" -> page,
          "path" -> path,
          "outputDir" -> pageOutputDir,
          "rootPath" -> rootPath,
          "allPipelines" -> pipelines,
          "allModules" -> modules,
          "allSamples" -> samples,
          "allLibraries" -> libraries
        )

      // Generating subpages
Peter van 't Hof's avatar
Peter van 't Hof committed
279
      val jobs = page.subPages.flatMap {
Peter van 't Hof's avatar
Peter van 't Hof committed
280
        case (name, subPage) => generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs)
Peter van 't Hof's avatar
Peter van 't Hof committed
281
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
282
283

      val renderFuture = Future {
Peter van 't Hof's avatar
Peter van 't Hof committed
284
285
286
        val file = new File(pageOutputDir, "index.html")
        logger.info(s"Start rendering: $file")

Peter van 't Hof's avatar
Peter van 't Hof committed
287
288
289
290
291
292
        val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
          pageArgs ++ Map("args" -> pageArgs))

        val writer = new PrintWriter(file)
        writer.println(output)
        writer.close()
Peter van 't Hof's avatar
Peter van 't Hof committed
293
294
        logger.info(s"Done rendering: $file")

Peter van 't Hof's avatar
Peter van 't Hof committed
295
296
      }

Peter van 't Hof's avatar
Peter van 't Hof committed
297
      renderFuture :: jobs
298
299
    }

300
  }
301

Peter van 't Hof's avatar
Peter van 't Hof committed
302
  def pipelineName: String
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322

  /** Files page, can be used general or at sample level */
  def filesPage(sampleId: Option[Int] = None, libraryId: Option[Int] = None): Future[ReportPage] = {
    val dbFiles = summary.getFiles(runId, sample = sampleId.map(SampleId),
      library = libraryId.map(LibraryId))
      .map(_.groupBy(_.pipelineId))
    val modulePages = dbFiles.map(_.map {
      case (pipelineId, files) =>
        val moduleSections = files.groupBy(_.moduleId).map {
          case (moduleId, files) =>
            val moduleName: Future[String] = moduleId match {
              case Some(id) => summary.getModuleName(pipelineId, id).map(_.getOrElse("Pipeline"))
              case _        => Future("Pipeline")
            }
            moduleName.map(_ -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> files)))
        }
        val moduleSectionsSorted = moduleSections.find(_._1 == "Pipeline") ++ moduleSections.filter(_._1 != "Pipeline")
        summary.getPipelineName(pipelineId = pipelineId).map(_.get -> Future(ReportPage(Nil, Await.result(Future.sequence(moduleSectionsSorted), Duration.Inf).toList, Map())))
    })

Peter van 't Hof's avatar
Peter van 't Hof committed
323
324
325
326
    val pipelineFiles = summary.getPipelineId(runId, pipelineName)
      .flatMap(pipelinelineId => dbFiles
        .map(x => x.get(pipelinelineId.get).getOrElse(Seq())
          .filter(_.moduleId.isEmpty)))
327
328
329
330
331
332
333
334
335
336
337
338

    modulePages.flatMap(Future.sequence(_)).map(x => ReportPage(x.toList,
      s"$pipelineName files" -> ReportSection("/nl/lumc/sasc/biopet/core/report/files.ssp", Map("files" -> Await.result(pipelineFiles, Duration.Inf))) ::
        "Sub pipelines/modules" -> ReportSection("/nl/lumc/sasc/biopet/core/report/fileModules.ssp", Map("pipelineIds" -> Await.result(dbFiles.map(_.keys.toList), Duration.Inf))) :: Nil, Map()))
  }

  /** This generate general pages that all reports should have */
  def generalPages(sampleId: Option[Int], libId: Option[Int]): List[(String, Future[ReportPage])] = List(
    "Versions" -> Future(ReportPage(List(), List("Executables" -> ReportSection("/nl/lumc/sasc/biopet/core/report/executables.ssp")), Map())),
    "Files" -> filesPage(sampleId, libId)
  )

339
}
Peter van 't Hof's avatar
Peter van 't Hof committed
340
341
342

object ReportBuilder {

Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
343
344
  implicit lazy val ec = ExecutionContext.global

345
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
346
  protected val engine = new TemplateEngine()
347
  engine.allowReload = false
348

349
  /** This will give the total number of pages including all nested pages */
Peter van 't Hof's avatar
Peter van 't Hof committed
350
351
352
  //  def countPages(page: ReportPage): Int = {
  //    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  //  }
353

Peter van 't Hof's avatar
Peter van 't Hof committed
354
355
356
357
358
359
360
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
361
    Logging.logger.debug("Rendering: " + location)
Peter van 't Hof's avatar
Peter van 't Hof committed
362

363
    engine.layout(location, args)
364
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
365
}