ReportBuilder.scala 10.4 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1 2 3 4 5 6 7 8 9 10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12 13 14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15 16
package nl.lumc.sasc.biopet.core.report

17
import java.io._
18

19
import nl.lumc.sasc.biopet.core.ToolCommandFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{ Library, Module, Pipeline, Sample }
21
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import org.broadinstitute.gatk.utils.commandline.Input
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.fusesource.scalate.{ TemplateEngine, TemplateSource }
25

Peter van 't Hof's avatar
Peter van 't Hof committed
26
import scala.collection.mutable
Peter van 't Hof's avatar
Peter van 't Hof committed
27
import scala.concurrent.{ Await, Future }
28
import scala.concurrent.duration.Duration
29
import scala.language.postfixOps
Peter van 't Hof's avatar
Peter van 't Hof committed
30
import scala.language.implicitConversions
31 32

/**
33 34 35
 * This trait is meant to make an extension for a report object
 *
 * @author pjvan_thof
36
 */
37
trait ReportBuilderExtension extends ToolCommandFunction {
38

39
  /** Report builder object */
40
  def builder: ReportBuilder
41

Peter van 't Hof's avatar
Peter van 't Hof committed
42 43
  def toolObject = builder

44
  @Input(required = true)
45 46 47
  var summaryDbFile: File = _

  var runId: Option[Int] = None
48

49
  /** OutputDir for the report  */
50 51
  var outputDir: File = _

52
  /** Arguments that are passed on the commandline */
53 54
  var args: Map[String, String] = Map()

55 56
  override def defaultCoreMemory = 4.0
  override def defaultThreads = 3
57

58 59
  override def beforeGraph(): Unit = {
    super.beforeGraph()
60 61 62 63
    jobOutputFile = new File(outputDir, ".report.log.out")
    javaMainClass = builder.getClass.getName.takeWhile(_ != '$')
  }

64
  /** Command to generate the report */
65 66
  override def cmdLine: String = {
    super.cmdLine +
67 68
      required("--summaryDb", summaryDbFile) +
      optional("--runId", runId) +
69
      required("--outputDir", outputDir) +
Peter van 't Hof's avatar
Peter van 't Hof committed
70
      args.map(x => required("-a", x._1 + "=" + x._2)).mkString
71 72 73
  }
}

74 75
trait ReportBuilder extends ToolCommand {

Peter van 't Hof's avatar
Peter van 't Hof committed
76 77
  implicit def toOption[T](x: T): Option[T] = Option(x)
  implicit def autoWait[T](x: Future[T]): T = Await.result(x, Duration.Inf)
78

79
  case class Args(summaryDbFile: File = null,
80
                  outputDir: File = null,
81
                  runId: Int = 0,
82
                  pageArgs: mutable.Map[String, Any] = mutable.Map()) extends AbstractArgs
83 84

  class OptParser extends AbstractOptParser {
85 86 87 88 89 90 91

    head(
      s"""
         |$commandName - Generate HTML formatted report from a biopet summary.json
       """.stripMargin
    )

92 93
    opt[File]('s', "summaryDb") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
      c.copy(summaryDbFile = x)
94 95 96 97
    } validate {
      x => if (x.exists) success else failure("Summary JSON file not found!")
    } text "Biopet summary JSON file"

Peter van 't Hof's avatar
Peter van 't Hof committed
98
    opt[File]('o', "outputDir") unbounded () required () maxOccurs 1 valueName "<file>" action { (x, c) =>
99
      c.copy(outputDir = x)
100 101
    } text "Output HTML report files to this directory"

102 103 104 105
    opt[Int]("runId") unbounded () maxOccurs 1 valueName "<int>" action { (x, c) =>
      c.copy(runId = x)
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
106
    opt[Map[String, String]]('a', "args") unbounded () action { (x, c) =>
107 108
      c.copy(pageArgs = c.pageArgs ++ x)
    }
109 110
  }

111
  /** summary object internaly */
112
  private var setSummary: SummaryDb = _
113

114
  /** Retrival of summary, read only */
115 116
  final def summary = setSummary

117 118 119 120
  private var setRunId: Int = 0

  final def runId = setRunId

Peter van 't Hof's avatar
Peter van 't Hof committed
121 122 123 124
  private var _setPipelines = Seq[Pipeline]()
  final def pipelines = _setPipelines
  private var _setModules = Seq[Module]()
  final def modules = _setModules
Peter van 't Hof's avatar
Peter van 't Hof committed
125 126 127 128
  private var _setSamples = Seq[Sample]()
  final def samples = _setSamples
  private var _setLibraries = Seq[Library]()
  final def libraries = _setLibraries
Peter van 't Hof's avatar
Peter van 't Hof committed
129

130
  /** default args that are passed to all page withing the report */
131 132
  def pageArgs: Map[String, Any] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
133 134 135
  private var done = 0
  private var total = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
136
  private var _sampleId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
137
  protected[report] def sampleId = _sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
138
  private var _libId: Option[Int] = None
Peter van 't Hof's avatar
Peter van 't Hof committed
139
  protected[report] def libId = _libId
Peter van 't Hof's avatar
Peter van 't Hof committed
140

141 142
  case class ExtFile(resourcePath: String, targetPath: String)

143 144 145 146 147 148 149 150
  def extFiles = List(
    "css/bootstrap_dashboard.css",
    "css/bootstrap.min.css",
    "css/bootstrap-theme.min.css",
    "css/sortable-theme-bootstrap.css",
    "js/jquery.min.js",
    "js/sortable.min.js",
    "js/bootstrap.min.js",
Peter van 't Hof's avatar
Peter van 't Hof committed
151
    "js/d3.v3.5.5.min.js",
152 153 154
    "fonts/glyphicons-halflings-regular.woff",
    "fonts/glyphicons-halflings-regular.ttf",
    "fonts/glyphicons-halflings-regular.woff2"
Peter van 't Hof's avatar
Peter van 't Hof committed
155
  ).map(x => ExtFile("/nl/lumc/sasc/biopet/core/report/ext/" + x, x))
156

157
  /** Main function to for building the report */
158 159 160 161
  def main(args: Array[String]): Unit = {
    logger.info("Start")

    val argsParser = new OptParser
Peter van 't Hof's avatar
Peter van 't Hof committed
162
    val cmdArgs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
163 164 165 166

    require(cmdArgs.outputDir.exists(), "Output dir does not exist")
    require(cmdArgs.outputDir.isDirectory, "Output dir is not a directory")

167
    setSummary = SummaryDb.openReadOnlySqliteSummary(cmdArgs.summaryDbFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
168 169
    setRunId = cmdArgs.runId

Peter van 't Hof's avatar
Peter van 't Hof committed
170
    cmdArgs.pageArgs.get("sampleId") match {
171
      case Some(s: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
172 173
        _sampleId = Await.result(summary.getSampleId(runId, s), Duration.Inf)
        cmdArgs.pageArgs += "sampleId" -> sampleId
Peter van 't Hof's avatar
Peter van 't Hof committed
174
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
175 176 177
    }

    cmdArgs.pageArgs.get("libId") match {
178
      case Some(l: String) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
179 180
        _libId = Await.result(summary.getLibraryId(runId, sampleId.get, l), Duration.Inf)
        cmdArgs.pageArgs += "libId" -> libId
Peter van 't Hof's avatar
Peter van 't Hof committed
181
      case _ =>
Peter van 't Hof's avatar
Peter van 't Hof committed
182 183
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
184 185
    _setPipelines = Await.result(summary.getPipelines(runId = Some(runId)), Duration.Inf)
    _setModules = Await.result(summary.getModules(runId = Some(runId)), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
186 187 188
    _setSamples = Await.result(summary.getSamples(runId = Some(runId), sampleId = sampleId), Duration.Inf)
    _setLibraries = Await.result(summary.getLibraries(runId = Some(runId), sampleId = sampleId, libId = libId), Duration.Inf)

189 190
    logger.info("Copy Base files")

Peter van 't Hof's avatar
Peter van 't Hof committed
191
    // Static files that will be copied to the output folder, then file is added to [resourceDir] it's need to be added here also
192 193
    val extOutputDir: File = new File(cmdArgs.outputDir, "ext")

194 195 196 197 198 199 200 201
    // Copy each resource files out to the report destination
    extFiles.par.foreach(
      resource =>
        IoUtils.copyStreamToFile(
          getClass.getResourceAsStream(resource.resourcePath),
          new File(extOutputDir, resource.targetPath),
          createDirs = true)
    )
202

203
    total = ReportBuilder.countPages(indexPage)
Peter van 't Hof's avatar
Peter van 't Hof committed
204 205
    logger.info(total + " pages to be generated")

Peter van 't Hof's avatar
Peter van 't Hof committed
206 207
    done = 0

Peter van 't Hof's avatar
Peter van 't Hof committed
208
    logger.info("Generate pages")
Peter van 't Hof's avatar
Peter van 't Hof committed
209
    val jobs = generatePage(summary, indexPage, cmdArgs.outputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
210
      args = pageArgs ++ cmdArgs.pageArgs.toMap ++
211
        Map("summary" -> summary, "reportName" -> reportName, "indexPage" -> indexPage, "runId" -> cmdArgs.runId))
212

Peter van 't Hof's avatar
Peter van 't Hof committed
213
    logger.info(jobs + " Done")
214 215
  }

216
  /** This must be implemented, this will be the root page of the report */
217 218
  def indexPage: ReportPage

219
  /** This must be implemented, this will become the title of the report */
220 221
  def reportName: String

222 223
  /**
   * This method will render the page and the subpages recursivly
Peter van 't Hof's avatar
Peter van 't Hof committed
224
   *
225 226 227 228 229 230 231
   * @param summary The summary object
   * @param page Page to render
   * @param outputDir Root output dir of the report
   * @param path Path from root to current page
   * @param args Args to add to this sub page, are args from current page are passed automaticly
   * @return Number of pages including all subpages that are rendered
   */
232
  def generatePage(summary: SummaryDb,
233 234 235
                   page: ReportPage,
                   outputDir: File,
                   path: List[String] = Nil,
Peter van 't Hof's avatar
Peter van 't Hof committed
236
                   args: Map[String, Any] = Map()): Int = {
237

Peter van 't Hof's avatar
Peter van 't Hof committed
238 239
    val pageOutputDir = new File(outputDir, path.mkString(File.separator))
    pageOutputDir.mkdirs()
240
    val rootPath = "./" + Array.fill(path.size)("../").mkString
241 242 243 244
    val pageArgs = args ++ page.args ++
      Map("page" -> page,
        "path" -> path,
        "outputDir" -> pageOutputDir,
Peter van 't Hof's avatar
Peter van 't Hof committed
245 246 247 248 249
        "rootPath" -> rootPath,
        "allPipelines" -> pipelines,
        "allModules" -> modules,
        "allSamples" -> samples,
        "allLibraries" -> libraries
250
      )
251

252
    // Generating subpages
253 254 255
    val jobs = page.subPages.par.flatMap {
      case (name, subPage) => Some(generatePage(summary, subPage, outputDir, path ::: name :: Nil, pageArgs))
      case _               => None
256 257
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
258 259 260 261
    val output = ReportBuilder.renderTemplate("/nl/lumc/sasc/biopet/core/report/main.ssp",
      pageArgs ++ Map("args" -> pageArgs))

    val file = new File(pageOutputDir, "index.html")
262 263 264 265
    val writer = new PrintWriter(file)
    writer.println(output)
    writer.close()

Peter van 't Hof's avatar
Peter van 't Hof committed
266 267
    done += 1
    if (done % 100 == 0) logger.info(done + " Done, " + (done.toDouble / total * 100) + "%")
268
    jobs.sum + 1
269 270
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
271 272 273

object ReportBuilder {

274
  /** Single template render engine, this will have a cache for all compile templates */
Peter van 't Hof's avatar
Peter van 't Hof committed
275 276
  protected val engine = new TemplateEngine()

277
  /** Cache of temp file for templates from the classpath / jar */
Peter van 't Hof's avatar
Peter van 't Hof committed
278
  private[report] var templateCache: Map[String, File] = Map()
279

280 281 282 283 284
  /** This will give the total number of pages including all nested pages */
  def countPages(page: ReportPage): Int = {
    page.subPages.map(x => countPages(x._2)).fold(1)(_ + _)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
285 286 287 288 289 290 291
  /**
   * This method will render a template that is located in the classpath / jar
   * @param location location in the classpath / jar
   * @param args Additional arguments, not required
   * @return Rendered result of template
   */
  def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
Peter van 't Hof's avatar
Peter van 't Hof committed
292 293
    Logging.logger.info("Rendering: " + location)

294 295
    val templateFile: File = templateCache.get(location) match {
      case Some(template) => template
296
      case _ =>
297
        val tempFile = File.createTempFile("ssp-template", new File(location).getName)
298
        tempFile.deleteOnExit()
299
        IoUtils.copyStreamToFile(getClass.getResourceAsStream(location), tempFile)
300 301 302 303 304
        templateCache += location -> tempFile
        tempFile
    }
    engine.layout(TemplateSource.fromFile(templateFile), args)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
305
}