BiopetCommandLineFunctionTrait.scala 6.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
package nl.lumc.sasc.biopet.core

18
//import java.io.BufferedInputStream
19
import java.io.File
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.core.config.Configurable
Peter van 't Hof's avatar
Peter van 't Hof committed
21
22
import org.broadinstitute.gatk.queue.QException
import org.broadinstitute.gatk.queue.function.CommandLineFunction
23
import org.broadinstitute.gatk.utils.commandline.{ Input, Argument }
24
//import scala.io.Source
25
import scala.sys.process.{ Process, ProcessLogger }
26
import scala.util.matching.Regex
27
28
import java.io.FileInputStream
import java.security.MessageDigest
29
30

trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurable {
Peter van 't Hof's avatar
Peter van 't Hof committed
31
  analysisName = configName
bow's avatar
bow committed
32
33

  @Input(doc = "deps", required = false)
34
  var deps: List[File] = Nil
bow's avatar
bow committed
35
36

  @Argument(doc = "Threads", required = false)
37
38
  var threads = 0
  val defaultThreads = 1
bow's avatar
bow committed
39
40

  @Argument(doc = "Vmem", required = false)
Peter van 't Hof's avatar
Peter van 't Hof committed
41
  var vmem: Option[String] = None
42
  val defaultVmem: String = ""
bow's avatar
bow committed
43

Peter van 't Hof's avatar
Peter van 't Hof committed
44
  @Argument(doc = "Executable", required = false)
Peter van 't Hof's avatar
Peter van 't Hof committed
45
  var executable: String = _
bow's avatar
bow committed
46

47
  protected[core] def beforeCmd {
48
  }
bow's avatar
bow committed
49

50
  protected[core] def afterGraph {
51
  }
bow's avatar
bow committed
52

53
  override def freezeFieldValues() {
Peter van 't Hof's avatar
Peter van 't Hof committed
54
    checkExecutable
55
    afterGraph
56
    if (jobOutputFile == null) jobOutputFile = new File(firstOutput.getParent + "/." + firstOutput.getName + "." + configName + ".out")
bow's avatar
bow committed
57

58
59
    if (threads == 0) threads = getThreads(defaultThreads)
    if (threads > 1) nCoresRequest = Option(threads)
bow's avatar
bow committed
60

Peter van 't Hof's avatar
Peter van 't Hof committed
61
    if (vmem.isEmpty) {
Peter van 't Hof's avatar
Peter van 't Hof committed
62
      vmem = config("vmem")
63
      if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
64
65
    }
    if (vmem != null) jobResourceRequests :+= "h_vmem=" + vmem
66
    jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
bow's avatar
bow committed
67

68
69
    super.freezeFieldValues()
  }
bow's avatar
bow committed
70

Peter van 't Hof's avatar
Peter van 't Hof committed
71
  protected def checkExecutable {
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
      try if (executable != null) {
        if (!BiopetCommandLineFunctionTrait.executableCache.contains(executable)) {
          val oldExecutable = executable
          val buffer = new StringBuffer()
          val cmd = Seq("which", executable)
          val process = Process(cmd).run(ProcessLogger(buffer.append(_)))
          if (process.exitValue == 0) {
            executable = buffer.toString
            val file = new File(executable)
            executable = file.getCanonicalPath
          } else {
            logger.error("executable: '" + executable + "' not found, please check config")
            throw new QException("executable: '" + executable + "' not found, please check config")
          }
          BiopetCommandLineFunctionTrait.executableCache += oldExecutable -> executable
          BiopetCommandLineFunctionTrait.executableCache += executable -> executable
        } else {
          executable = BiopetCommandLineFunctionTrait.executableCache(executable)
        }

        if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
          val is = new FileInputStream(executable)
          val cnt = is.available
          val bytes = Array.ofDim[Byte](cnt)
          is.read(bytes)
          is.close()
          val temp = MessageDigest.getInstance("MD5").digest(bytes).map("%02X".format(_)).mkString.toLowerCase
          BiopetCommandLineFunctionTrait.executableMd5Cache += executable -> temp
        }
      } catch {
        case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
104
105
      }
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
106
107
108
    val md5 = BiopetCommandLineFunctionTrait.executableMd5Cache(executable)
    if (md5 == null) addJobReportBinding("md5sum_exe", md5)
    else addJobReportBinding("md5sum_exe", "None")
109
  }
bow's avatar
bow committed
110

111
  final protected def preCmdInternal {
Peter van 't Hof's avatar
Peter van 't Hof committed
112
    checkExecutable
bow's avatar
bow committed
113

114
    beforeCmd
bow's avatar
bow committed
115

116
    addJobReportBinding("cores", if (nCoresRequest.get.toInt > 0) nCoresRequest.get.toInt else 1)
117
    addJobReportBinding("version", getVersion)
118
  }
bow's avatar
bow committed
119

120
  protected def versionCommand: String = null
121
  protected val versionRegex: Regex = null
122
  protected val versionExitcode = List(0) // Can select multiple
Peter van 't Hof's avatar
Peter van 't Hof committed
123
  private def getVersionInternal: String = {
124
    if (versionCommand == null || versionRegex == null) return "N/A"
125
126
    val stdout = new StringBuffer()
    val stderr = new StringBuffer()
127
128
129
130
    def outputLog = "Version command: \n" + versionCommand +
      "\n output log: \n stdout: \n" + stdout.toString +
      "\n stderr: \n" + stderr.toString
    val process = Process(versionCommand).run(ProcessLogger(stdout append _ + "\n", stderr append _ + "\n"))
131
    if (!versionExitcode.contains(process.exitValue)) {
132
      logger.warn("getVersion give exit code " + process.exitValue + ", version not found \n" + outputLog)
133
134
      return "N/A"
    }
135
    for (line <- stdout.toString.split("\n") ++ stderr.toString.split("\n")) {
bow's avatar
bow committed
136
      line match {
137
        case versionRegex(m) => return m
Peter van 't Hof's avatar
Peter van 't Hof committed
138
        case _               =>
139
140
      }
    }
141
    logger.warn("getVersion give a exit code " + process.exitValue + " but no version was found, executable correct? \n" + outputLog)
142
143
    return "N/A"
  }
bow's avatar
bow committed
144

Peter van 't Hof's avatar
Peter van 't Hof committed
145
146
147
148
149
150
  def getVersion: String = {
    if (!BiopetCommandLineFunctionTrait.versionCache.contains(executable))
      BiopetCommandLineFunctionTrait.versionCache += executable -> getVersionInternal
    return BiopetCommandLineFunctionTrait.versionCache(executable)
  }

bow's avatar
bow committed
151
152
153
  def getThreads(default: Int): Int = {
    val maxThreads: Int = config("maxthreads", default = 8)
    val threads: Int = config("threads", default = default)
154
155
156
    if (maxThreads > threads) return threads
    else return maxThreads
  }
bow's avatar
bow committed
157
158
159
160

  def getThreads(default: Int, module: String): Int = {
    val maxThreads: Int = config("maxthreads", default = 8, submodule = module)
    val threads: Int = config("threads", default = default, submodule = module)
161
162
163
164
    if (maxThreads > threads) return threads
    else return maxThreads
  }
}
Peter van 't Hof's avatar
Peter van 't Hof committed
165
166
167
168

object BiopetCommandLineFunctionTrait {
  import scala.collection.mutable.Map
  private val versionCache: Map[String, String] = Map()
169
170
  private val executableMd5Cache: Map[String, String] = Map()
  private val executableCache: Map[String, String] = Map()
Peter van 't Hof's avatar
Peter van 't Hof committed
171
}