ReportBuilder.scala 9.89 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Sample }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import org.broadinstitute.gatk.utils.commandline.Input
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import nl.lumc.sasc.biopet.utils.tryToParseNumber
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import org.fusesource.scalate.{ TemplateEngine, TemplateSource }
26

Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.collection.mutable
Peter van 't Hof's avatar
Peter van 't Hof committed
28
import scala.concurrent.{ Await, Future }
29
import scala.concurrent.duration.Duration
30
import scala.language.postfixOps
31
32

/**
33
34
35
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
36
 */
37
trait ReportBuilderExtension extends ToolCommandFunction {
38

39
  /** Report builder object */
40
  def builder: ReportBuilder
41

Peter van 't Hof's avatar
Peter van 't Hof committed
42
43
  def toolObject = builder

44
  @Input(required = true)
45
46
47
  var summaryDbFile: File = _

  var runId: Option[Int] = None
48

49
  /** OutputDir for the report  */
50
51
  var outputDir: File = _

52
  /** Arguments that are passed on the commandline */
53
54
  var args: Map[String, String] = Map()

55
56
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
57

58
59
  override def beforeGraph(): Unit = {
    super.beforeGraph()
60
61
62
63
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

64
  /** Command to generate the report */
65
66
  override def cmdLine: String = {
    super.cmdLine +
67
68
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
69
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
70
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
71
72
73
  }
}

74
75
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
Peter van 't Hof committed
76
77
  implicit def toOption[T](x: T): Option[T] = Option(x)
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
78

79
  case class Args(summaryDbFile: File = null,
80
                  outputDir: File = null,
81
                  runId: Int = 0,
82
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
83
84

  class OptParser extends AbstractOptParser {
85
86
87
88
89
90
91

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

92
93
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
94
95
96
97
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
98
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
99
      c.copy(outputDir = x)
100
101
    } text "Output HTML report files to this directory"

102
103
104
105
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
106
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
107
108
      c.copy(pageArgs = c.pageArgs ++ x)
    }
109
110
  }

111
  /** summary object internaly */
112
  private var setSummary: SummaryDb = _
113

114
  /** Retrival of summary, read only */
115
116
  final def summary = setSummary

117
118
119
120
  private var setRunId: Int = 0

  final def runId = setRunId

Peter van 't Hof's avatar
Peter van 't Hof committed
121
122
123
124
  private var _setSamples = Seq[Sample]()
  final def samples = _setSamples
  private var _setLibraries = Seq[Library]()
  final def libraries = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
125

126
  /** default args that are passed to all page withing the report */
127
128
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
129
130
131
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
132
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
133
  protected[report] def sampleId = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
134
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
135
  protected[report] def libId = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
136

137
138
  case class ExtFile(resourcePath: String, targetPath: String)

139
140
141
142
143
144
145
146
  def extFiles = List(
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
147
    "js/d3.v3.5.5.min.js",
148
149
150
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
151
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
152

153
  /** Main function to for building the report */
154
155
156
157
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
158
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
159
160
161
162

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

Peter van 't Hof's avatar
Peter van 't Hof committed
163
164
165
    setSummary = SummaryDb.openSqliteSummary(cmdArgs.summaryDbFile)
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
166
    cmdArgs.pageArgs.get("sampleId") match {
167
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
168
169
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
170
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
171
172
173
    }

    cmdArgs.pageArgs.get("libId") match {
174
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
175
176
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
177
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
178
179
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
180
181
182
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

183
184
    logger.info("Copy Base files")

Peter van 't Hof's avatar
Peter van 't Hof committed
185
    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
186
187
    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")

188
189
190
191
192
193
194
195
    // Copy each resource files out to the report destination
    extFiles.par.foreach(
      resource =>
        IoUtils.copyStreamToFile(
          getClass.getResourceAsStream(resource.resourcePath),
          new File(extOutputDir, resource.targetPath),
          createDirs = true)
    )
196

197
    total = ReportBuilder.countPages(indexPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
198
199
    logger.info(total + " pages to be generated")

Peter van 't Hof's avatar
Peter van 't Hof committed
200
201
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
202
    logger.info("Generate pages")
Peter van 't Hof's avatar
Peter van 't Hof committed
203
    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
204
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
205
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
206

Peter van 't Hof's avatar
Peter van 't Hof committed
207
    logger.info(jobs + " Done")
208
209
  }

210
  /** This must be implemented, this will be the root page of the report */
211
212
  def indexPage: ReportPage

213
  /** This must be implemented, this will become the title of the report */
214
215
  def reportName: String

216
217
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
218
   *
219
220
221
222
223
224
225
   * @param summary The summary object
   * @param page Page to render
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
226
  def generatePage(summary: SummaryDb,
227
228
229
                   page: ReportPage,
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
Peter van 't Hof committed
230
                   args: Map[String, Any] = Map()): Int = {
231

Peter van 't Hof's avatar
Peter van 't Hof committed
232
233
    val pageOutputDir = new File(outputDir, path.mkString(File.separator))
    pageOutputDir.mkdirs()
234
    val rootPath = "./" + Array.fill(path.size)("../").mkString
235
236
237
238
239
240
    val pageArgs = args ++ page.args ++
      Map("page" -> page,
        "path" -> path,
        "outputDir" -> pageOutputDir,
        "rootPath" -> rootPath
      )
241

242
    // Generating subpages
243
244
245
    val jobs = page.subPages.par.flatMap {
      case (name, subPage) => Some(generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs))
      case _               => None
246
247
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
248
249
250
251
    val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
      pageArgs ++ Map("args" -> pageArgs))

    val file = new File(pageOutputDir, "index.html")
252
253
254
255
    val writer = new PrintWriter(file)
    writer.println(output)
    writer.close()

Peter van 't Hof's avatar
Peter van 't Hof committed
256
257
    done += 1
    if (done % 100 == 0) logger.info(done + " Done, " + (done.toDouble / total * 100) + "%")
258
    jobs.sum + 1
259
260
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
261
262
263

object ReportBuilder {

264
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
265
266
  protected val engine = new TemplateEngine()

267
  /** Cache of temp file for templates from the classpath / jar */
Peter van 't Hof's avatar
Peter van 't Hof committed
268
  private[report] var templateCache: Map[String, File] = Map()
269

270
271
272
273
274
  /** This will give the total number of pages including all nested pages */
  def countPages(page: ReportPage): Int = {
    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
275
276
277
278
279
280
281
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
282
283
    Logging.logger.info("Rendering: " + location)

284
285
    val templateFile: File = templateCache.get(location) match {
      case Some(template) => template
286
      case _ =>
287
        val tempFile = File.createTempFile("ssp-template", new File(location).getName)
288
        tempFile.deleteOnExit()
289
        IoUtils.copyStreamToFile(getClass.getResourceAsStream(location), tempFile)
290
291
292
293
294
        templateCache += location -> tempFile
        tempFile
    }
    engine.layout(TemplateSource.fromFile(templateFile), args)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
295
}