ReportBuilder.scala 10.1 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import org.broadinstitute.gatk.utils.commandline.Input
24
import org.fusesource.scalate.TemplateEngine
25

Peter van 't Hof's avatar
Peter van 't Hof committed
26
import scala.collection.mutable
Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.concurrent.{ Await, ExecutionContextExecutor, Future }
28
import scala.concurrent.duration.Duration
29
import scala.language.postfixOps
Peter van 't Hof's avatar
Peter van 't Hof committed
30
import scala.language.implicitConversions
31
32

/**
33
34
35
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
36
 */
37
trait ReportBuilderExtension extends ToolCommandFunction {
38

39
  /** Report builder object */
40
  def builder: ReportBuilder
41

Peter van 't Hof's avatar
Peter van 't Hof committed
42
43
  def toolObject = builder

44
  @Input(required = true)
45
46
47
  var summaryDbFile: File = _

  var runId: Option[Int] = None
48

49
  /** OutputDir for the report  */
50
51
  var outputDir: File = _

52
  /** Arguments that are passed on the commandline */
53
54
  var args: Map[String, String] = Map()

55
56
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
57

58
59
  override def beforeGraph(): Unit = {
    super.beforeGraph()
60
61
62
63
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

64
  /** Command to generate the report */
65
66
  override def cmdLine: String = {
    super.cmdLine +
67
68
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
69
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
70
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
71
72
73
  }
}

74
75
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
Peter van 't Hof committed
76
  implicit lazy val global: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global
Peter van 't Hof's avatar
Peter van 't Hof committed
77
  implicit def toOption[T](x: T): Option[T] = Option(x)
Peter van 't Hof's avatar
Peter van 't Hof committed
78
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
79

80
  case class Args(summaryDbFile: File = null,
81
                  outputDir: File = null,
82
                  runId: Int = 0,
83
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
84
85

  class OptParser extends AbstractOptParser {
86
87
88
89
90
91
92

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

93
94
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
95
96
97
98
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
99
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
100
      c.copy(outputDir = x)
101
102
    } text "Output HTML report files to this directory"

103
104
105
106
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
107
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
108
109
      c.copy(pageArgs = c.pageArgs ++ x)
    }
110
111
  }

112
  /** summary object internaly */
113
  private var setSummary: SummaryDb = _
114

115
  /** Retrival of summary, read only */
116
117
  final def summary = setSummary

118
119
120
121
  private var setRunId: Int = 0

  final def runId = setRunId

Peter van 't Hof's avatar
Peter van 't Hof committed
122
123
124
125
  private var _setPipelines = Seq[Pipeline]()
  final def pipelines = _setPipelines
  private var _setModules = Seq[Module]()
  final def modules = _setModules
Peter van 't Hof's avatar
Peter van 't Hof committed
126
127
128
129
  private var _setSamples = Seq[Sample]()
  final def samples = _setSamples
  private var _setLibraries = Seq[Library]()
  final def libraries = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
130

131
  /** default args that are passed to all page withing the report */
132
133
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
134
135
136
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
137
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
138
  protected[report] def sampleId = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
139
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
140
  protected[report] def libId = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
141

142
143
  case class ExtFile(resourcePath: String, targetPath: String)

144
145
146
147
148
149
150
151
  def extFiles = List(
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
152
    "js/d3.v3.5.5.min.js",
153
154
155
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
156
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
157

158
  /** Main function to for building the report */
159
160
161
162
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
163
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
164
165
166
167

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

168
    setSummary = SummaryDb.openReadOnlySqliteSummary(cmdArgs.summaryDbFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
169
170
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
171
    cmdArgs.pageArgs.get("sampleId") match {
172
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
173
174
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
175
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
176
177
178
    }

    cmdArgs.pageArgs.get("libId") match {
179
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
180
181
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
182
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
183
184
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
185
186
    _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
    _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
187
188
189
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
190
    // TODO: switch to future for base files
191
192
    logger.info("Copy Base files")

Peter van 't Hof's avatar
Peter van 't Hof committed
193
    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
194
195
    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")

196
197
198
199
200
201
202
203
    // Copy each resource files out to the report destination
    extFiles.par.foreach(
      resource =>
        IoUtils.copyStreamToFile(
          getClass.getResourceAsStream(resource.resourcePath),
          new File(extOutputDir, resource.targetPath),
          createDirs = true)
    )
204

Peter van 't Hof's avatar
Peter van 't Hof committed
205
206
207
    val rootPage = indexPage

    //    total = ReportBuilder.countPages(rootPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
208
209
    logger.info(total + " pages to be generated")

Peter van 't Hof's avatar
Peter van 't Hof committed
210
211
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
212
    logger.info("Generate pages")
Peter van 't Hof's avatar
Peter van 't Hof committed
213
    val jobs = generatePage(summary, rootPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
214
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
Peter van 't Hof's avatar
Peter van 't Hof committed
215
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> rootPage, "runId" -> cmdArgs.runId))
216

Peter van 't Hof's avatar
Peter van 't Hof committed
217
    Await.result(jobs, Duration.Inf)
218
219
  }

220
  /** This must be implemented, this will be the root page of the report */
Peter van 't Hof's avatar
Peter van 't Hof committed
221
  def indexPage: Future[ReportPage]
222

223
  /** This must be implemented, this will become the title of the report */
224
225
  def reportName: String

226
227
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
228
   *
229
   * @param summary The summary object
Peter van 't Hof's avatar
Peter van 't Hof committed
230
   * @param pageFuture Page to render
231
232
233
234
235
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
236
  def generatePage(summary: SummaryDb,
Peter van 't Hof's avatar
Peter van 't Hof committed
237
                   pageFuture: Future[ReportPage],
238
239
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
Peter van 't Hof committed
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
                   args: Map[String, Any] = Map()): Future[_] = {
    pageFuture.flatMap { page =>
      val pageOutputDir = new File(outputDir, path.mkString(File.separator))
      pageOutputDir.mkdirs()
      val rootPath = "./" + Array.fill(path.size)("../").mkString
      val pageArgs = args ++ page.args ++
        Map("page" -> page,
          "path" -> path,
          "outputDir" -> pageOutputDir,
          "rootPath" -> rootPath,
          "allPipelines" -> pipelines,
          "allModules" -> modules,
          "allSamples" -> samples,
          "allLibraries" -> libraries
        )

      // Generating subpages
      val jobs = Future.sequence(page.subPages.map {
        case (name, subPage) => generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs)
      })

      val renderFuture = Future {
        val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
          pageArgs ++ Map("args" -> pageArgs))

        val file = new File(pageOutputDir, "index.html")
        val writer = new PrintWriter(file)
        writer.println(output)
        writer.close()
      }

      Future.sequence(renderFuture :: jobs :: Nil)
272
273
    }

274
275
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
276
277
278

object ReportBuilder {

279
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
280
  protected val engine = new TemplateEngine()
281
  engine.allowReload = false
282

283
  /** This will give the total number of pages including all nested pages */
Peter van 't Hof's avatar
Peter van 't Hof committed
284
285
286
  //  def countPages(page: ReportPage): Int = {
  //    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  //  }
287

Peter van 't Hof's avatar
Peter van 't Hof committed
288
289
290
291
292
293
294
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
295
296
    Logging.logger.info("Rendering: " + location)

297
    engine.layout(location, args)
298
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
299
}