BiopetCommandLineFunctionTrait.scala 7.62 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
18
package nl.lumc.sasc.biopet.core

import java.io.File
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.core.config.Configurable
Peter van 't Hof's avatar
Peter van 't Hof committed
20
21
import org.broadinstitute.gatk.queue.QException
import org.broadinstitute.gatk.queue.function.CommandLineFunction
22
23
import org.broadinstitute.gatk.utils.commandline.{ Input, Argument }
import scala.sys.process.{ Process, ProcessLogger }
24
import scala.util.matching.Regex
25
26
import java.io.FileInputStream
import java.security.MessageDigest
27

Peter van 't Hof's avatar
Peter van 't Hof committed
28
29
30
/**
 * Biopet command line trait to auto check executable and cluster values
 */
31
trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurable {
Peter van 't Hof's avatar
Peter van 't Hof committed
32
  analysisName = configName
bow's avatar
bow committed
33
34

  @Input(doc = "deps", required = false)
35
  var deps: List[File] = Nil
bow's avatar
bow committed
36

Peter van 't Hof's avatar
Peter van 't Hof committed
37

38
39
  var threads = 0
  val defaultThreads = 1
bow's avatar
bow committed
40

Peter van 't Hof's avatar
Peter van 't Hof committed
41
  var vmem: Option[String] = None
42
  val defaultVmem: String = ""
Peter van 't Hof's avatar
Peter van 't Hof committed
43
  var executable: String = _
bow's avatar
bow committed
44

Peter van 't Hof's avatar
Peter van 't Hof committed
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
  /**
   * Can override this method. This is executed just before the job is ready to run.
   * Can check on run time files from pipeline here
   */
  protected[core] def beforeCmd { }

  /**
   * Can override this method. This is executed after the script is done en queue starts to generate the graph
   */
  protected[core] def afterGraph { }
  //TODO: function need rename to beforeGraph

  /**
   * Set default output file, threads and vmem for current job
   */
60
  override def freezeFieldValues() {
Peter van 't Hof's avatar
Peter van 't Hof committed
61
    checkExecutable
62
    afterGraph
63
    if (jobOutputFile == null) jobOutputFile = new File(firstOutput.getParent + "/." + firstOutput.getName + "." + configName + ".out")
bow's avatar
bow committed
64

65
66
    if (threads == 0) threads = getThreads(defaultThreads)
    if (threads > 1) nCoresRequest = Option(threads)
bow's avatar
bow committed
67

Peter van 't Hof's avatar
Peter van 't Hof committed
68
    if (vmem.isEmpty) {
Peter van 't Hof's avatar
Peter van 't Hof committed
69
      vmem = config("vmem")
70
      if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
71
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
72
    if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
73
    jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
bow's avatar
bow committed
74

75
76
    super.freezeFieldValues()
  }
bow's avatar
bow committed
77

Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
80
  /**
   * Checks executable. Follow full CanonicalPath, checks if it is existing and do a md5sum on it to store in job report
   */
81
  protected[core] def checkExecutable {
82
83
84
85
86
87
88
89
90
91
92
93
    if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
      try if (executable != null) {
        if (!BiopetCommandLineFunctionTrait.executableCache.contains(executable)) {
          val oldExecutable = executable
          val buffer = new StringBuffer()
          val cmd = Seq("which", executable)
          val process = Process(cmd).run(ProcessLogger(buffer.append(_)))
          if (process.exitValue == 0) {
            executable = buffer.toString
            val file = new File(executable)
            executable = file.getCanonicalPath
          } else {
94
            BiopetQScript.addError("executable: '" + executable + "' not found, please check config")
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
          }
          BiopetCommandLineFunctionTrait.executableCache += oldExecutable -> executable
          BiopetCommandLineFunctionTrait.executableCache += executable -> executable
        } else {
          executable = BiopetCommandLineFunctionTrait.executableCache(executable)
        }

        if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
          val is = new FileInputStream(executable)
          val cnt = is.available
          val bytes = Array.ofDim[Byte](cnt)
          is.read(bytes)
          is.close()
          val temp = MessageDigest.getInstance("MD5").digest(bytes).map("%02X".format(_)).mkString.toLowerCase
          BiopetCommandLineFunctionTrait.executableMd5Cache += executable -> temp
        }
      } catch {
        case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
113
114
      }
    }
115
116
    val md5 = BiopetCommandLineFunctionTrait.executableMd5Cache.get(executable)
    addJobReportBinding("md5sum_exe", md5.getOrElse("None"))
117
  }
bow's avatar
bow committed
118

Peter van 't Hof's avatar
Peter van 't Hof committed
119
120
121
  /**
   * executes checkExecutable method and fill job report
   */
122
  final protected def preCmdInternal {
Peter van 't Hof's avatar
Peter van 't Hof committed
123
    checkExecutable
bow's avatar
bow committed
124

125
    beforeCmd
bow's avatar
bow committed
126

127
    addJobReportBinding("cores", if (nCoresRequest.get.toInt > 0) nCoresRequest.get.toInt else 1)
128
    addJobReportBinding("version", getVersion)
129
  }
bow's avatar
bow committed
130

Peter van 't Hof's avatar
Peter van 't Hof committed
131
132
133
134
  /**
   * Command to get version of executable
   * @return
   */
135
  protected def versionCommand: String = null
Peter van 't Hof's avatar
Peter van 't Hof committed
136
137

  /** Regex to get version from version command output */
138
  protected val versionRegex: Regex = null
Peter van 't Hof's avatar
Peter van 't Hof committed
139
140
141
142
143

  /** Allowed exit codes for the version command */
  protected val versionExitcode = List(0)

  /** Executes the version command */
Peter van 't Hof's avatar
Peter van 't Hof committed
144
  private def getVersionInternal: String = {
145
    if (versionCommand == null || versionRegex == null) return "N/A"
146
    val exe = new File(versionCommand.trim.split(" ")(0))
Peter van 't Hof's avatar
Peter van 't Hof committed
147
    if (!exe.exists()) return "N/A"
148
149
    val stdout = new StringBuffer()
    val stderr = new StringBuffer()
150
151
152
153
    def outputLog = "Version command: \n" + versionCommand +
      "\n output log: \n stdout: \n" + stdout.toString +
      "\n stderr: \n" + stderr.toString
    val process = Process(versionCommand).run(ProcessLogger(stdout append _ + "\n", stderr append _ + "\n"))
154
    if (!versionExitcode.contains(process.exitValue)) {
155
      logger.warn("getVersion give exit code " + process.exitValue + ", version not found \n" + outputLog)
156
157
      return "N/A"
    }
158
    for (line <- stdout.toString.split("\n") ++ stderr.toString.split("\n")) {
bow's avatar
bow committed
159
      line match {
160
        case versionRegex(m) => return m
Peter van 't Hof's avatar
Peter van 't Hof committed
161
        case _               =>
162
163
      }
    }
164
    logger.warn("getVersion give a exit code " + process.exitValue + " but no version was found, executable correct? \n" + outputLog)
165
166
    return "N/A"
  }
bow's avatar
bow committed
167

Peter van 't Hof's avatar
Peter van 't Hof committed
168
  /** Get version from cache otherwise execute the version command  */
Peter van 't Hof's avatar
Peter van 't Hof committed
169
170
171
172
173
174
  def getVersion: String = {
    if (!BiopetCommandLineFunctionTrait.versionCache.contains(executable))
      BiopetCommandLineFunctionTrait.versionCache += executable -> getVersionInternal
    return BiopetCommandLineFunctionTrait.versionCache(executable)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
175
176
177
178
179
  /**
   * Get threads from config
   * @param default default when not found in config
   * @return number of threads
   */
bow's avatar
bow committed
180
181
182
  def getThreads(default: Int): Int = {
    val maxThreads: Int = config("maxthreads", default = 8)
    val threads: Int = config("threads", default = default)
183
184
185
    if (maxThreads > threads) return threads
    else return maxThreads
  }
bow's avatar
bow committed
186

Peter van 't Hof's avatar
Peter van 't Hof committed
187
188
189
190
191
192
  /**
   * Get threads from config
   * @param default default when not found in config
   * @param module Module when this is difrent from default
   * @return number of threads
   */
bow's avatar
bow committed
193
194
195
  def getThreads(default: Int, module: String): Int = {
    val maxThreads: Int = config("maxthreads", default = 8, submodule = module)
    val threads: Int = config("threads", default = default, submodule = module)
196
197
198
199
    if (maxThreads > threads) return threads
    else return maxThreads
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
200

Peter van 't Hof's avatar
Peter van 't Hof committed
201
202
203
/**
 * stores global caches
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
204
205
206
object BiopetCommandLineFunctionTrait {
  import scala.collection.mutable.Map
  private val versionCache: Map[String, String] = Map()
207
208
  private val executableMd5Cache: Map[String, String] = Map()
  private val executableCache: Map[String, String] = Map()
209
}