BiopetQScript.scala 4.94 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
18
package nl.lumc.sasc.biopet.core

import java.io.File
19
20
21

import nl.lumc.sasc.biopet.core.config.Configurable
import nl.lumc.sasc.biopet.core.report.ReportBuilderExtension
Peter van 't Hof's avatar
Peter van 't Hof committed
22
23
import org.broadinstitute.gatk.queue.QSettings
import org.broadinstitute.gatk.queue.function.QFunction
24
import org.broadinstitute.gatk.queue.function.scattergather.ScatterGatherableFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
26
27
import org.broadinstitute.gatk.utils.commandline.Argument

28
import scala.collection.mutable.ListBuffer
29

Peter van 't Hof's avatar
Peter van 't Hof committed
30
/** Base for biopet pipeline */
31
trait BiopetQScript extends Configurable with GatkLogging {
32

33
  @Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
34
  val configfiles: List[File] = Nil
bow's avatar
bow committed
35

36
  @Argument(doc = "Config values, value should be formatted like 'key=value' or 'path:path:key=value'", fullName = "config_value", shortName = "cv", required = false)
37
38
  val configValues: List[String] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
39
  /** Output directory of pipeline */
Peter van 't Hof's avatar
Peter van 't Hof committed
40
  var outputDir: File = {
Peter van 't Hof's avatar
Peter van 't Hof committed
41
42
    if (config.contains("output_dir", path = Nil)) config("output_dir", path = Nil).asFile
    else new File(".")
43
  }
bow's avatar
bow committed
44

45
  @Argument(doc = "Disable all scatters", shortName = "DSC", required = false)
46
  var disableScatter: Boolean = false
47

bow's avatar
bow committed
48
49
  var outputFiles: Map[String, File] = Map()

50
51
52
53
  protected case class InputFile(file: File, md5: Option[String] = None)

  var inputFiles: List[InputFile] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
54
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
55
  var qSettings: QSettings
bow's avatar
bow committed
56

Peter van 't Hof's avatar
Peter van 't Hof committed
57
58
59
60
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  var functions: Seq[QFunction]

  /** Init for pipeline */
61
  def init()
bow's avatar
bow committed
62

Peter van 't Hof's avatar
Peter van 't Hof committed
63
  /** Pipeline itself */
64
  def biopetScript()
Peter van 't Hof's avatar
Peter van 't Hof committed
65

66
67
68
  /** Returns the extension to make the report */
  def reportClass: Option[ReportBuilderExtension] = None

Peter van 't Hof's avatar
Peter van 't Hof committed
69
  /** Script from queue itself, final to force some checks for each pipeline and write report */
70
  final def script() {
Peter van 't Hof's avatar
Peter van 't Hof committed
71
72
    outputDir = config("output_dir")
    outputDir = outputDir.getAbsoluteFile
73
74
    init()
    biopetScript()
75

76
    if (disableScatter) for (function <- functions) function match {
77
78
79
      case f: ScatterGatherableFunction => f.scatterCount = 1
      case _                            =>
    }
80
    for (function <- functions) function match {
81
82
83
      case f: BiopetCommandLineFunctionTrait =>
        f.preProcessExecutable()
        f.beforeGraph()
Peter van 't Hof's avatar
Peter van 't Hof committed
84
        f.commandLine
Peter van 't Hof's avatar
Peter van 't Hof committed
85
      case _ =>
86
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
87

Peter van 't Hof's avatar
Peter van 't Hof committed
88
89
90
    if (outputDir.getParentFile.canWrite || (outputDir.exists && outputDir.canWrite))
      globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
    else BiopetQScript.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, outputdir can not be created")
91

92
93
94
95
96
    inputFiles.foreach{ i =>
      if (!i.file.exists()) BiopetQScript.addError(s"Input file does not exist: ${i.file}")
      if (!i.file.canRead()) BiopetQScript.addError(s"Input file can not be read: ${i.file}")
    }

97
98
    reportClass.foreach(add(_))

99
    BiopetQScript.checkErrors()
100
  }
bow's avatar
bow committed
101

Peter van 't Hof's avatar
Peter van 't Hof committed
102
103
104
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  def add(functions: QFunction*)

105
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
106
  def addAll(functions: scala.Traversable[org.broadinstitute.gatk.queue.function.QFunction])
107

108
  /** Function to set isIntermediate and add in 1 line */
bow's avatar
bow committed
109
  def add(function: QFunction, isIntermediate: Boolean = false) {
110
111
112
    function.isIntermediate = isIntermediate
    add(function)
  }
113
}
114
115
116
117

object BiopetQScript extends Logging {
  private val errors: ListBuffer[Exception] = ListBuffer()

118
119
  def addError(error: String, debug: String = null): Unit = {
    val msg = error + (if (debug != null && logger.isDebugEnabled) "; " + debug else "")
120
121
122
    errors.append(new Exception(msg))
  }

123
124
  protected def checkErrors(): Unit = {
    if (errors.nonEmpty) {
125
126
127
128
129
130
131
132
      logger.error("*************************")
      logger.error("Biopet found some errors:")
      if (logger.isDebugEnabled) {
        for (e <- errors) {
          logger.error(e.getMessage)
          logger.debug(e.getStackTrace.mkString("Stack trace:\n", "\n", "\n"))
        }
      } else {
Peter van 't Hof's avatar
Peter van 't Hof committed
133
        errors.map(_.getMessage).sorted.distinct.foreach(logger.error(_))
134
135
136
137
      }
      throw new IllegalStateException("Biopet found errors")
    }
  }
138
}