BiopetQScript.scala 7.59 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
17
package nl.lumc.sasc.biopet.core

import java.io.File
18

Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.core.summary.{ SummaryQScript, WriteSummary }
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.utils.config.Configurable
21
import nl.lumc.sasc.biopet.core.report.ReportBuilderExtension
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.core.workaround.BiopetQCommandLine
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import nl.lumc.sasc.biopet.utils.Logging
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.broadinstitute.gatk.queue.{ QScript, QSettings }
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import org.broadinstitute.gatk.queue.function.QFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
26
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
27

Peter van 't Hof's avatar
Peter van 't Hof committed
28
/** Base for biopet pipeline */
29
trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
30

31
  @Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
32
  val configfiles: List[File] = Nil
bow's avatar
bow committed
33

Sander Bollen's avatar
Sander Bollen committed
34
  @Argument(doc = "Config values, value should be formatted like 'key=value' or 'namespace:namespace:key=value'", fullName = "config_value", shortName = "cv", required = false)
35
36
  val configValues: List[String] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
37
  /** Output directory of pipeline */
Peter van 't Hof's avatar
Peter van 't Hof committed
38
  var outputDir: File = {
Peter van 't Hof's avatar
Peter van 't Hof committed
39
40
    if (config.contains("output_dir", path = Nil)) config("output_dir", path = Nil).asFile
    else new File(".")
41
  }
bow's avatar
bow committed
42

43
  @Argument(doc = "Disable all scatters", shortName = "DSC", required = false)
44
  var disableScatter: Boolean = false
45

bow's avatar
bow committed
46
47
  var outputFiles: Map[String, File] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
48
  type InputFile = BiopetQScript.InputFile
49
50
51

  var inputFiles: List[InputFile] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
52
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
53
  var qSettings: QSettings
bow's avatar
bow committed
54

Peter van 't Hof's avatar
Peter van 't Hof committed
55
56
57
58
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  var functions: Seq[QFunction]

  /** Init for pipeline */
59
  def init()
bow's avatar
bow committed
60

Peter van 't Hof's avatar
Peter van 't Hof committed
61
  /** Pipeline itself */
62
  def biopetScript()
Peter van 't Hof's avatar
Peter van 't Hof committed
63

64
65
66
  /** Returns the extension to make the report */
  def reportClass: Option[ReportBuilderExtension] = None

67
68
  val skipWriteDependencies: Boolean = config("skip_write_dependencies", default = false)

Peter van 't Hof's avatar
Peter van 't Hof committed
69
  /** Script from queue itself, final to force some checks for each pipeline and write report */
70
  final def script() {
Peter van 't Hof's avatar
Peter van 't Hof committed
71
72
    outputDir = config("output_dir")
    outputDir = outputDir.getAbsoluteFile
73
74
75

    BiopetQScript.checkOutputDir(outputDir)

76
77
    init()
    biopetScript()
Peter van 't Hof's avatar
Peter van 't Hof committed
78
    logger.info("Biopet script done")
79

Peter van 't Hof's avatar
Peter van 't Hof committed
80
81
82
83
84
85
    if (disableScatter) {
      logger.info("Disable scatters")
      for (function <- functions) function match {
        case f: ScatterGatherableFunction => f.scatterCount = 1
        case _                            =>
      }
86
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
87
88
89
90

    this match {
      case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
        logger.info("Write report is skipped because sample flag is used")
91
92
93
      case _ => reportClass.foreach { report =>
        add(report)
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
94
95
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
96
    logger.info("Running pre commands")
97
    for (function <- functions) function match {
98
      case f: BiopetCommandLineFunction =>
99
100
        f.preProcessExecutable()
        f.beforeGraph()
Peter van 't Hof's avatar
Peter van 't Hof committed
101
        f.internalBeforeGraph()
Peter van 't Hof's avatar
Peter van 't Hof committed
102
        f.commandLine
Peter van 't Hof's avatar
Peter van 't Hof committed
103
104
      case f: WriteSummary => f.init()
      case _               =>
105
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
106

Peter van 't Hof's avatar
Peter van 't Hof committed
107
    val logDir = new File(outputDir, ".log" + File.separator + qSettings.runName.toLowerCase)
Peter van 't Hof's avatar
Peter van 't Hof committed
108

Peter van 't Hof's avatar
Peter van 't Hof committed
109
    if (outputDir.getParentFile.canWrite || (outputDir.exists && outputDir.canWrite))
Peter van 't Hof's avatar
Peter van 't Hof committed
110
      globalConfig.writeReport(new File(logDir, "config"))
Sander Bollen's avatar
Sander Bollen committed
111
    else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, output directory cannot be created")
112

Peter van 't Hof's avatar
Peter van 't Hof committed
113
114
    logger.info("Checking input files")
    inputFiles.par.foreach { i =>
Peter van 't Hof's avatar
Peter van 't Hof committed
115
      if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
116
      if (!i.file.canRead) Logging.addError(s"Input file can not be read: ${i.file}")
Sander Bollen's avatar
Sander Bollen committed
117
      if (!i.file.isAbsolute) Logging.addError(s"Input file should be an absolute path: ${i.file}")
118
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
119

Peter van 't Hof's avatar
Peter van 't Hof committed
120
    functions.filter(_.jobOutputFile == null).foreach(f => {
Peter van 't Hof's avatar
Peter van 't Hof committed
121
      try {
Peter van 't Hof's avatar
Peter van 't Hof committed
122
123
        val className = if (f.getClass.isAnonymousClass) f.getClass.getSuperclass.getSimpleName else f.getClass.getSimpleName
        f.jobOutputFile = new File(f.firstOutput.getAbsoluteFile.getParent, "." + f.firstOutput.getName + "." + className + ".out")
Peter van 't Hof's avatar
Peter van 't Hof committed
124
125
126
      } catch {
        case e: NullPointerException => logger.warn(s"Can't generate a jobOutputFile for $f")
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
127
    })
128

Peter van 't Hof's avatar
Peter van 't Hof committed
129
130
    if (!skipWriteDependencies) WriteDependencies.writeDependencies(
      functions,
Peter van 't Hof's avatar
Peter van 't Hof committed
131
      new File(logDir, "graph"))
132

Peter van 't Hof's avatar
Peter van 't Hof committed
133
    Logging.checkErrors()
Peter van 't Hof's avatar
Peter van 't Hof committed
134
    logger.info("Script complete without errors")
135
  }
bow's avatar
bow committed
136

Peter van 't Hof's avatar
Peter van 't Hof committed
137
138
139
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  def add(functions: QFunction*)

140
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
141
  def addAll(functions: scala.Traversable[org.broadinstitute.gatk.queue.function.QFunction])
142

143
  /** Function to set isIntermediate and add in 1 line */
bow's avatar
bow committed
144
  def add(function: QFunction, isIntermediate: Boolean = false) {
145
146
147
    function.isIntermediate = isIntermediate
    add(function)
  }
148
149
150
151
152
153
154
155
156

  def add(subPipeline: QScript): Unit = {
    subPipeline.qSettings = this.qSettings
    subPipeline match {
      case that: SummaryQScript =>
        that.init()
        that.biopetScript()
        this match {
          case s: SummaryQScript => s.addSummaryQScript(that)
Peter van 't Hof's avatar
Peter van 't Hof committed
157
          case _                 =>
158
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
159
      case that: BiopetQScript =>
160
161
162
163
164
165
        that.init()
        that.biopetScript()
      case _ => subPipeline.script
    }
    addAll(subPipeline.functions)
  }
166
}
Peter van 't Hof's avatar
Peter van 't Hof committed
167
168

object BiopetQScript {
Peter van 't Hof's avatar
Peter van 't Hof committed
169
  case class InputFile(file: File, md5: Option[String] = None)
170
171
172

  def checkOutputDir(outputDir: File): Unit = {
    // Sanity checks
Peter van 't Hof's avatar
Peter van 't Hof committed
173
174
    require(outputDir.getAbsoluteFile.getParentFile.canRead, s"No premision to read parent of outputdir: ${outputDir.getParentFile}")
    require(outputDir.getAbsoluteFile.getParentFile.canWrite, s"No premision to write parent of outputdir: ${outputDir.getParentFile}")
175
    outputDir.mkdir()
Peter van 't Hof's avatar
Peter van 't Hof committed
176
177
    require(outputDir.getAbsoluteFile.canRead, s"No premision to read outputdir: $outputDir")
    require(outputDir.getAbsoluteFile.canWrite, s"No premision to write outputdir: $outputDir")
178
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
179

Peter van 't Hof's avatar
Peter van 't Hof committed
180
  def safeInputs(function: QFunction): Option[Seq[File]] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
181
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
182
      Some(function.inputs)
Peter van 't Hof's avatar
Peter van 't Hof committed
183
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
184
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
185
186
187
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
188
  def safeOutputs(function: QFunction): Option[Seq[File]] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
189
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
190
      Some(function.outputs)
Peter van 't Hof's avatar
Peter van 't Hof committed
191
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
192
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
193
194
195
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
196
  def safeDoneFiles(function: QFunction): Option[Seq[File]] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
197
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
198
      Some(function.doneOutputs)
Peter van 't Hof's avatar
Peter van 't Hof committed
199
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
200
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
201
202
203
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
204
  def safeFailFiles(function: QFunction): Option[Seq[File]] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
205
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
206
      Some(function.failOutputs)
Peter van 't Hof's avatar
Peter van 't Hof committed
207
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
208
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
209
210
211
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
212
  def safeIsDone(function: QFunction): Option[Boolean] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
213
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
214
      Some(function.isDone)
Peter van 't Hof's avatar
Peter van 't Hof committed
215
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
216
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
217
218
219
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
220
  def safeIsFail(function: QFunction): Option[Boolean] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
221
    try {
Peter van 't Hof's avatar
Peter van 't Hof committed
222
      Some(function.isFail)
Peter van 't Hof's avatar
Peter van 't Hof committed
223
    } catch {
Peter van 't Hof's avatar
Peter van 't Hof committed
224
      case e: NullPointerException => None
Peter van 't Hof's avatar
Peter van 't Hof committed
225
226
227
    }
  }

228
}