BiopetQScript.scala 5.43 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16 17 18
package nl.lumc.sasc.biopet.core

import java.io.File
19

20
import nl.lumc.sasc.biopet.core.summary.SummaryQScript
Peter van 't Hof's avatar
Peter van 't Hof committed
21
import nl.lumc.sasc.biopet.utils.config.Configurable
22
import nl.lumc.sasc.biopet.core.report.ReportBuilderExtension
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import nl.lumc.sasc.biopet.utils.Logging
24
import org.broadinstitute.gatk.queue.{ QScript, QSettings }
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import org.broadinstitute.gatk.queue.function.QFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
26
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
27

Peter van 't Hof's avatar
Peter van 't Hof committed
28
/** Base for biopet pipeline */
29
trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
30

31
  @Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
32
  val configfiles: List[File] = Nil
bow's avatar
bow committed
33

Sander Bollen's avatar
Sander Bollen committed
34
  @Argument(doc = "Config values, value should be formatted like 'key=value' or 'namespace:namespace:key=value'", fullName = "config_value", shortName = "cv", required = false)
35 36
  val configValues: List[String] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
37
  /** Output directory of pipeline */
Peter van 't Hof's avatar
Peter van 't Hof committed
38
  var outputDir: File = {
Peter van 't Hof's avatar
Peter van 't Hof committed
39 40
    if (config.contains("output_dir", path = Nil)) config("output_dir", path = Nil).asFile
    else new File(".")
41
  }
bow's avatar
bow committed
42

43
  @Argument(doc = "Disable all scatters", shortName = "DSC", required = false)
44
  var disableScatter: Boolean = false
45

bow's avatar
bow committed
46 47
  var outputFiles: Map[String, File] = Map()

Peter van 't Hof's avatar
Peter van 't Hof committed
48
  type InputFile = BiopetQScript.InputFile
49 50 51

  var inputFiles: List[InputFile] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
52
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
53
  var qSettings: QSettings
bow's avatar
bow committed
54

Peter van 't Hof's avatar
Peter van 't Hof committed
55 56 57 58
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  var functions: Seq[QFunction]

  /** Init for pipeline */
59
  def init()
bow's avatar
bow committed
60

Peter van 't Hof's avatar
Peter van 't Hof committed
61
  /** Pipeline itself */
62
  def biopetScript()
Peter van 't Hof's avatar
Peter van 't Hof committed
63

64 65 66
  /** Returns the extension to make the report */
  def reportClass: Option[ReportBuilderExtension] = None

Peter van 't Hof's avatar
Peter van 't Hof committed
67
  /** Script from queue itself, final to force some checks for each pipeline and write report */
68
  final def script() {
Peter van 't Hof's avatar
Peter van 't Hof committed
69 70
    outputDir = config("output_dir")
    outputDir = outputDir.getAbsoluteFile
71 72
    init()
    biopetScript()
73

74
    if (disableScatter) for (function <- functions) function match {
75 76 77
      case f: ScatterGatherableFunction => f.scatterCount = 1
      case _                            =>
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
78 79 80 81 82 83 84

    this match {
      case q: MultiSampleQScript if q.onlySamples.nonEmpty && !q.samples.forall(x => q.onlySamples.contains(x._1)) =>
        logger.info("Write report is skipped because sample flag is used")
      case _ => reportClass.foreach(add(_))
    }

85
    for (function <- functions) function match {
86
      case f: BiopetCommandLineFunction =>
87 88
        f.preProcessExecutable()
        f.beforeGraph()
Peter van 't Hof's avatar
Peter van 't Hof committed
89
        f.internalBeforeGraph()
Peter van 't Hof's avatar
Peter van 't Hof committed
90
        f.commandLine
Peter van 't Hof's avatar
Peter van 't Hof committed
91
      case _ =>
92
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
93

Peter van 't Hof's avatar
Peter van 't Hof committed
94 95
    if (outputDir.getParentFile.canWrite || (outputDir.exists && outputDir.canWrite))
      globalConfig.writeReport(qSettings.runName, new File(outputDir, ".log/" + qSettings.runName))
Sander Bollen's avatar
Sander Bollen committed
96
    else Logging.addError("Parent of output dir: '" + outputDir.getParent + "' is not writeable, output directory cannot be created")
97

Peter van 't Hof's avatar
Peter van 't Hof committed
98
    inputFiles.foreach { i =>
Peter van 't Hof's avatar
Peter van 't Hof committed
99
      if (!i.file.exists()) Logging.addError(s"Input file does not exist: ${i.file}")
100
      if (!i.file.canRead) Logging.addError(s"Input file can not be read: ${i.file}")
Sander Bollen's avatar
Sander Bollen committed
101
      if (!i.file.isAbsolute) Logging.addError(s"Input file should be an absolute path: ${i.file}")
102
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
103

Peter van 't Hof's avatar
Peter van 't Hof committed
104
    functions.filter(_.jobOutputFile == null).foreach(f => {
Peter van 't Hof's avatar
Peter van 't Hof committed
105
      try {
106
        f.jobOutputFile = new File(f.firstOutput.getAbsoluteFile.getParent, "." + f.firstOutput.getName + "." + f.getClass.getSimpleName + ".out")
Peter van 't Hof's avatar
Peter van 't Hof committed
107 108 109
      } catch {
        case e: NullPointerException => logger.warn(s"Can't generate a jobOutputFile for $f")
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
110
    })
111

112 113
    if (logger.isDebugEnabled) WriteDependencies.writeDependencies(functions, new File(outputDir, s".log/${qSettings.runName}.deps.json"))

Peter van 't Hof's avatar
Peter van 't Hof committed
114
    Logging.checkErrors()
115
  }
bow's avatar
bow committed
116

Peter van 't Hof's avatar
Peter van 't Hof committed
117 118 119
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
  def add(functions: QFunction*)

120
  /** Get implemented from org.broadinstitute.gatk.queue.QScript */
Peter van 't Hof's avatar
Peter van 't Hof committed
121
  def addAll(functions: scala.Traversable[org.broadinstitute.gatk.queue.function.QFunction])
122

123
  /** Function to set isIntermediate and add in 1 line */
bow's avatar
bow committed
124
  def add(function: QFunction, isIntermediate: Boolean = false) {
125 126 127
    function.isIntermediate = isIntermediate
    add(function)
  }
128 129 130 131 132 133 134 135 136

  def add(subPipeline: QScript): Unit = {
    subPipeline.qSettings = this.qSettings
    subPipeline match {
      case that: SummaryQScript =>
        that.init()
        that.biopetScript()
        this match {
          case s: SummaryQScript => s.addSummaryQScript(that)
Peter van 't Hof's avatar
Peter van 't Hof committed
137
          case _                 =>
138
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
139
      case that: BiopetQScript =>
140 141 142 143 144 145
        that.init()
        that.biopetScript()
      case _ => subPipeline.script
    }
    addAll(subPipeline.functions)
  }
146
}
Peter van 't Hof's avatar
Peter van 't Hof committed
147 148

object BiopetQScript {
Peter van 't Hof's avatar
Peter van 't Hof committed
149
  case class InputFile(file: File, md5: Option[String] = None)
150
}