BiopetCommandLineFunctionTrait.scala 6.54 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
18
package nl.lumc.sasc.biopet.core

import java.io.File
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.core.config.Configurable
Peter van 't Hof's avatar
Peter van 't Hof committed
20
21
import org.broadinstitute.gatk.queue.QException
import org.broadinstitute.gatk.queue.function.CommandLineFunction
22
23
import org.broadinstitute.gatk.utils.commandline.{ Input, Argument }
import scala.sys.process.{ Process, ProcessLogger }
24
import scala.util.matching.Regex
25
26
import java.io.FileInputStream
import java.security.MessageDigest
27
28

trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurable {
Peter van 't Hof's avatar
Peter van 't Hof committed
29
  analysisName = configName
bow's avatar
bow committed
30
31

  @Input(doc = "deps", required = false)
32
  var deps: List[File] = Nil
bow's avatar
bow committed
33
34

  @Argument(doc = "Threads", required = false)
35
36
  var threads = 0
  val defaultThreads = 1
bow's avatar
bow committed
37
38

  @Argument(doc = "Vmem", required = false)
Peter van 't Hof's avatar
Peter van 't Hof committed
39
  var vmem: Option[String] = None
40
  val defaultVmem: String = ""
bow's avatar
bow committed
41

Peter van 't Hof's avatar
Peter van 't Hof committed
42
  @Argument(doc = "Executable", required = false)
Peter van 't Hof's avatar
Peter van 't Hof committed
43
  var executable: String = _
bow's avatar
bow committed
44

45
  protected[core] def beforeCmd {
46
  }
bow's avatar
bow committed
47

48
  protected[core] def afterGraph {
49
  }
bow's avatar
bow committed
50

51
  override def freezeFieldValues() {
Peter van 't Hof's avatar
Peter van 't Hof committed
52
    checkExecutable
53
    afterGraph
54
    if (jobOutputFile == null) jobOutputFile = new File(firstOutput.getParent + "/." + firstOutput.getName + "." + configName + ".out")
bow's avatar
bow committed
55

56
57
    if (threads == 0) threads = getThreads(defaultThreads)
    if (threads > 1) nCoresRequest = Option(threads)
bow's avatar
bow committed
58

Peter van 't Hof's avatar
Peter van 't Hof committed
59
    if (vmem.isEmpty) {
Peter van 't Hof's avatar
Peter van 't Hof committed
60
      vmem = config("vmem")
61
      if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
62
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
63
    if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
64
    jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
bow's avatar
bow committed
65

66
67
    super.freezeFieldValues()
  }
bow's avatar
bow committed
68

Peter van 't Hof's avatar
Peter van 't Hof committed
69
  protected def checkExecutable {
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
      try if (executable != null) {
        if (!BiopetCommandLineFunctionTrait.executableCache.contains(executable)) {
          val oldExecutable = executable
          val buffer = new StringBuffer()
          val cmd = Seq("which", executable)
          val process = Process(cmd).run(ProcessLogger(buffer.append(_)))
          if (process.exitValue == 0) {
            executable = buffer.toString
            val file = new File(executable)
            executable = file.getCanonicalPath
          } else {
            logger.error("executable: '" + executable + "' not found, please check config")
            throw new QException("executable: '" + executable + "' not found, please check config")
          }
          BiopetCommandLineFunctionTrait.executableCache += oldExecutable -> executable
          BiopetCommandLineFunctionTrait.executableCache += executable -> executable
        } else {
          executable = BiopetCommandLineFunctionTrait.executableCache(executable)
        }

        if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
          val is = new FileInputStream(executable)
          val cnt = is.available
          val bytes = Array.ofDim[Byte](cnt)
          is.read(bytes)
          is.close()
          val temp = MessageDigest.getInstance("MD5").digest(bytes).map("%02X".format(_)).mkString.toLowerCase
          BiopetCommandLineFunctionTrait.executableMd5Cache += executable -> temp
        }
      } catch {
        case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
102
103
      }
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
104
105
106
    val md5 = BiopetCommandLineFunctionTrait.executableMd5Cache(executable)
    if (md5 == null) addJobReportBinding("md5sum_exe", md5)
    else addJobReportBinding("md5sum_exe", "None")
107
  }
bow's avatar
bow committed
108

109
  final protected def preCmdInternal {
Peter van 't Hof's avatar
Peter van 't Hof committed
110
    checkExecutable
bow's avatar
bow committed
111

112
    beforeCmd
bow's avatar
bow committed
113

114
    addJobReportBinding("cores", if (nCoresRequest.get.toInt > 0) nCoresRequest.get.toInt else 1)
115
    addJobReportBinding("version", getVersion)
116
  }
bow's avatar
bow committed
117

118
  protected def versionCommand: String = null
119
  protected val versionRegex: Regex = null
120
  protected val versionExitcode = List(0) // Can select multiple
Peter van 't Hof's avatar
Peter van 't Hof committed
121
  private def getVersionInternal: String = {
122
    if (versionCommand == null || versionRegex == null) return "N/A"
123
124
    val stdout = new StringBuffer()
    val stderr = new StringBuffer()
125
126
127
128
    def outputLog = "Version command: \n" + versionCommand +
      "\n output log: \n stdout: \n" + stdout.toString +
      "\n stderr: \n" + stderr.toString
    val process = Process(versionCommand).run(ProcessLogger(stdout append _ + "\n", stderr append _ + "\n"))
129
    if (!versionExitcode.contains(process.exitValue)) {
130
      logger.warn("getVersion give exit code " + process.exitValue + ", version not found \n" + outputLog)
131
132
      return "N/A"
    }
133
    for (line <- stdout.toString.split("\n") ++ stderr.toString.split("\n")) {
bow's avatar
bow committed
134
      line match {
135
        case versionRegex(m) => return m
Peter van 't Hof's avatar
Peter van 't Hof committed
136
        case _               =>
137
138
      }
    }
139
    logger.warn("getVersion give a exit code " + process.exitValue + " but no version was found, executable correct? \n" + outputLog)
140
141
    return "N/A"
  }
bow's avatar
bow committed
142

Peter van 't Hof's avatar
Peter van 't Hof committed
143
144
145
146
147
148
  def getVersion: String = {
    if (!BiopetCommandLineFunctionTrait.versionCache.contains(executable))
      BiopetCommandLineFunctionTrait.versionCache += executable -> getVersionInternal
    return BiopetCommandLineFunctionTrait.versionCache(executable)
  }

bow's avatar
bow committed
149
150
151
  def getThreads(default: Int): Int = {
    val maxThreads: Int = config("maxthreads", default = 8)
    val threads: Int = config("threads", default = default)
152
153
154
    if (maxThreads > threads) return threads
    else return maxThreads
  }
bow's avatar
bow committed
155
156
157
158

  def getThreads(default: Int, module: String): Int = {
    val maxThreads: Int = config("maxthreads", default = 8, submodule = module)
    val threads: Int = config("threads", default = default, submodule = module)
159
160
161
162
    if (maxThreads > threads) return threads
    else return maxThreads
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
163
164
165
166

object BiopetCommandLineFunctionTrait {
  import scala.collection.mutable.Map
  private val versionCache: Map[String, String] = Map()
167
168
  private val executableMd5Cache: Map[String, String] = Map()
  private val executableCache: Map[String, String] = Map()
Peter van 't Hof's avatar
Peter van 't Hof committed
169
}