Commit 5688a64d authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Extracted version in a trait

parent 755c54b0
......@@ -146,59 +146,6 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
case Some(n) if n > 0 => n
case _ => 1
})
addJobReportBinding("version", getVersion)
}
/** Command to get version of executable */
protected[core] def versionCommand: String = null
/** Regex to get version from version command output */
protected[core] def versionRegex: Regex = null
/** Allowed exit codes for the version command */
protected[core] def versionExitcode = List(0)
/** Executes the version command */
private[core] def getVersionInternal: Option[String] = {
if (versionCommand == null || versionRegex == null) None
else getVersionInternal(versionCommand, versionRegex)
}
/** Executes the version command */
private[core] def getVersionInternal(versionCommand: String, versionRegex: Regex): Option[String] = {
if (versionCommand == null || versionRegex == null) return None
val exe = new File(versionCommand.trim.split(" ")(0))
if (!exe.exists()) return None
val stdout = new StringBuffer()
val stderr = new StringBuffer()
def outputLog = "Version command: \n" + versionCommand +
"\n output log: \n stdout: \n" + stdout.toString +
"\n stderr: \n" + stderr.toString
val process = Process(versionCommand).run(ProcessLogger(stdout append _ + "\n", stderr append _ + "\n"))
if (!versionExitcode.contains(process.exitValue())) {
logger.warn("getVersion give exit code " + process.exitValue + ", version not found \n" + outputLog)
return None
}
for (line <- stdout.toString.split("\n") ++ stderr.toString.split("\n")) {
line match {
case versionRegex(m) => return Some(m)
case _ =>
}
}
logger.warn("getVersion give a exit code " + process.exitValue + " but no version was found, executable correct? \n" + outputLog)
None
}
/** Get version from cache otherwise execute the version command */
def getVersion: Option[String] = {
if (!BiopetCommandLineFunction.executableCache.contains(executable))
preProcessExecutable()
if (!BiopetCommandLineFunction.versionCache.contains(versionCommand))
getVersionInternal match {
case Some(version) => BiopetCommandLineFunction.versionCache += versionCommand -> version
case _ =>
}
BiopetCommandLineFunction.versionCache.get(versionCommand)
}
private[core] var _inputAsStdin = false
......@@ -284,7 +231,6 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
/** stores global caches */
object BiopetCommandLineFunction {
private[core] val versionCache: mutable.Map[String, String] = mutable.Map()
private[core] val executableMd5Cache: mutable.Map[String, String] = mutable.Map()
private[core] val executableCache: mutable.Map[String, String] = mutable.Map()
}
......@@ -55,12 +55,12 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
def getJavaVersion: Option[String] = {
if (!BiopetCommandLineFunction.executableCache.contains(executable))
preProcessExecutable()
if (!BiopetCommandLineFunction.versionCache.contains(javaVersionCommand))
getVersionInternal(javaVersionCommand, """java version "(.*)"""".r) match {
case Some(version) => BiopetCommandLineFunction.versionCache += javaVersionCommand -> version
if (!Version.versionCache.contains(javaVersionCommand))
Version.getVersionInternal(javaVersionCommand, """java version "(.*)"""".r) match {
case Some(version) => Version.versionCache += javaVersionCommand -> version
case _ =>
}
BiopetCommandLineFunction.versionCache.get(javaVersionCommand)
Version.versionCache.get(javaVersionCommand)
}
override def setupRetry(): Unit = {
......
......@@ -5,9 +5,12 @@ import nl.lumc.sasc.biopet.FullVersion
/**
* Created by pjvanthof on 11/09/15.
*/
trait ToolCommandFuntion extends BiopetJavaCommandLineFunction {
trait ToolCommandFuntion extends BiopetJavaCommandLineFunction with Version {
def toolObject: Object
def versionCommand = ""
def versionRegex = "".r
override def getVersion = Some("Biopet " + FullVersion)
override def beforeGraph(): Unit = {
......
package nl.lumc.sasc.biopet.core
import java.io.File
import nl.lumc.sasc.biopet.utils.Logging
import org.broadinstitute.gatk.queue.function.QFunction
import scala.collection.mutable
import scala.sys.process.{ Process, ProcessLogger }
import scala.util.matching.Regex
/**
* Created by pjvan_thof on 10/13/15.
*/
trait Version extends QFunction {
/** Command to get version of executable */
def versionCommand: String
/** Regex to get version from version command output */
def versionRegex: Regex
/** Allowed exit codes for the version command */
protected[core] def versionExitcode = List(0)
/** Executes the version command */
private[core] def getVersionInternal: Option[String] = {
if (versionCommand == null || versionRegex == null) None
else Version.getVersionInternal(versionCommand, versionRegex, versionExitcode)
}
/** Get version from cache otherwise execute the version command */
def getVersion: Option[String] = {
if (!Version.versionCache.contains(versionCommand))
getVersionInternal match {
case Some(version) => Version.versionCache += versionCommand -> version
case _ =>
}
Version.versionCache.get(versionCommand)
}
override def freezeFieldValues(): Unit = {
super.freezeFieldValues()
addJobReportBinding("version", getVersion.getOrElse("NA"))
}
}
object Version extends Logging {
private[core] val versionCache: mutable.Map[String, String] = mutable.Map()
/** Executes the version command */
private[core] def getVersionInternal(versionCommand: String,
versionRegex: Regex,
versionExitcode: List[Int] = List(0)): Option[String] = {
if (versionCache.contains(versionCommand)) return versionCache.get(versionCommand)
else if (versionCommand == null || versionRegex == null) return None
else {
val exe = new File(versionCommand.trim.split(" ")(0))
if (!exe.exists()) return None
val stdout = new StringBuffer()
val stderr = new StringBuffer()
def outputLog = "Version command: \n" + versionCommand +
"\n output log: \n stdout: \n" + stdout.toString +
"\n stderr: \n" + stderr.toString
val process = Process(versionCommand).run(ProcessLogger(stdout append _ + "\n", stderr append _ + "\n"))
if (!versionExitcode.contains(process.exitValue())) {
logger.warn("getVersion give exit code " + process.exitValue + ", version not found \n" + outputLog)
return None
}
for (line <- stdout.toString.split("\n") ++ stderr.toString.split("\n")) {
line match {
case versionRegex(m) => return Some(m)
case _ =>
}
}
logger.warn("getVersion give a exit code " + process.exitValue + " but no version was found, executable correct? \n" + outputLog)
None
}
}
}
\ No newline at end of file
......@@ -17,12 +17,12 @@ package nl.lumc.sasc.biopet.core.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
/** Extension for md5sum */
class Md5sum(val root: Configurable) extends BiopetCommandLineFunction {
class Md5sum(val root: Configurable) extends BiopetCommandLineFunction with Version {
@Input(doc = "Input")
var input: File = _
......@@ -31,8 +31,8 @@ class Md5sum(val root: Configurable) extends BiopetCommandLineFunction {
executable = config("exe", default = "md5sum")
override def versionRegex = """md5sum \(GNU coreutils\) (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """md5sum \(GNU coreutils\) (.*)""".r
def versionCommand = executable + " --version"
/** return commandline to execute */
def cmdLine = required(executable) + required(input) + " > " + required(output)
......
......@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.core.summary
import java.io.{ File, PrintWriter }
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, BiopetJavaCommandLineFunction, SampleLibraryTag }
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction, BiopetJavaCommandLineFunction, SampleLibraryTag }
import nl.lumc.sasc.biopet.utils.ConfigUtils
import nl.lumc.sasc.biopet.{ LastCommitHash, Version }
import org.broadinstitute.gatk.queue.function.{ InProcessFunction, QFunction }
......@@ -73,15 +73,17 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
val executables: Map[String, Any] = {
(for (f <- qscript.functions if f.isInstanceOf[BiopetCommandLineFunction]) yield {
f match {
case f: BiopetJavaCommandLineFunction =>
case f: BiopetJavaCommandLineFunction with Version =>
f.configName -> Map("version" -> f.getVersion.getOrElse(None),
"java_md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"java_version" -> f.getJavaVersion,
"jar_path" -> f.jarFile)
case f: BiopetCommandLineFunction =>
case f: BiopetCommandLineFunction with Version =>
f.configName -> Map("version" -> f.getVersion.getOrElse(None),
"md5" -> BiopetCommandLineFunction.executableMd5Cache.getOrElse(f.executable, None),
"path" -> f.executable)
case f: Configurable with Version =>
f.configName -> Map("version" -> f.getVersion.getOrElse(None))
case _ => throw new IllegalStateException("This should not be possible")
}
......@@ -113,7 +115,7 @@ class WriteSummary(val root: Configurable) extends InProcessFunction with Config
}).foldRight(jobsMap)((a, b) => ConfigUtils.mergeMaps(a, b)) ++
Map("meta" -> Map(
"last_commit_hash" -> LastCommitHash,
"pipeline_version" -> Version,
"pipeline_version" -> nl.lumc.sasc.biopet.Version,
"pipeline_name" -> qscript.summaryName,
"output_dir" -> qscript.outputDir,
"run_name" -> config("run_name", default = qSettings.runName).asString,
......
......@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Reference }
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction, Reference }
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
/**
......@@ -26,7 +26,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
*
* Based on version 1.1.1
*/
class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Reference {
class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Reference with Version {
@Input(doc = "Fastq file R1", shortName = "R1")
var R1: File = null
......@@ -40,9 +40,9 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Refe
var output: File = null
executable = config("exe", default = "bowtie", freeVar = false)
override def versionRegex = """.*[Vv]ersion:? (.*)""".r
def versionRegex = """.*[Vv]ersion:? (.*)""".r
override def versionExitcode = List(0, 1)
override def versionCommand = executable + " --version"
def versionCommand = executable + " --version"
override def defaultCoreMemory = 4.0
override def defaultThreads = 8
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
* Wrapper for the cufflinks command line tool.
* Written based on cufflinks version v2.2.1 (md5: 07c831c4f8b4e161882731ea5694ff80)
*/
class Cufflinks(val root: Configurable) extends BiopetCommandLineFunction {
class Cufflinks(val root: Configurable) extends BiopetCommandLineFunction with Version {
/** default executable */
executable = config("exe", default = "cufflinks")
......@@ -185,8 +185,8 @@ class Cufflinks(val root: Configurable) extends BiopetCommandLineFunction {
/** do not contact server to check for update availability [FALSE] */
var no_update_check: Boolean = config("no_update_check", default = false)
override def versionRegex = """cufflinks v(.*)""".r
override def versionCommand = executable
def versionRegex = """cufflinks v(.*)""".r
def versionCommand = executable
override def versionExitcode = List(0, 1)
def cmdLine =
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
* Wrapper for the cuffquant command line tool.
* Written based on cuffquant version v2.2.1 (md5: 0765b82b11db9256f5be341a7da884d6)
*/
class Cuffquant(val root: Configurable) extends BiopetCommandLineFunction {
class Cuffquant(val root: Configurable) extends BiopetCommandLineFunction with Version {
/** default executable */
executable = config("exe", default = "cuffquant")
......@@ -117,8 +117,8 @@ class Cuffquant(val root: Configurable) extends BiopetCommandLineFunction {
/** Disable SCV correction */
var no_scv_correction: Boolean = config("no_scv_correction", default = false)
override def versionRegex = """cuffquant v(.*)""".r
override def versionCommand = executable
def versionRegex = """cuffquant v(.*)""".r
def versionCommand = executable
override def versionExitcode = List(0, 1)
def cmdLine =
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.summary.Summarizable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -29,7 +29,7 @@ import scala.io.Source
* Extension for cutadept
* Based on version 1.5
*/
class Cutadapt(val root: Configurable) extends BiopetCommandLineFunction with Summarizable {
class Cutadapt(val root: Configurable) extends BiopetCommandLineFunction with Summarizable with Version {
@Input(doc = "Input fastq file")
var fastq_input: File = _
......@@ -40,8 +40,8 @@ class Cutadapt(val root: Configurable) extends BiopetCommandLineFunction with Su
var stats_output: File = _
executable = config("exe", default = "cutadapt")
override def versionCommand = executable + " --version"
override def versionRegex = """(.*)""".r
def versionCommand = executable + " --version"
def versionRegex = """(.*)""".r
var default_clip_mode: String = config("default_clip_mode", default = "3")
var opt_adapter: Set[String] = config("adapter", default = Nil)
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
* Extension for fastqc
* Based on version 0.10.1 and 0.11.2
*/
class Fastqc(val root: Configurable) extends BiopetCommandLineFunction {
class Fastqc(val root: Configurable) extends BiopetCommandLineFunction with Version {
@Input(doc = "Contaminants", required = false)
var contaminants: Option[File] = None
......@@ -47,8 +47,8 @@ class Fastqc(val root: Configurable) extends BiopetCommandLineFunction {
var nogroup: Boolean = config("nogroup", default = false)
var extract: Boolean = config("extract", default = true)
override def versionRegex = """FastQC (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """FastQC (.*)""".r
def versionCommand = executable + " --version"
override def defaultThreads = 4
/** Sets contaminants and adapters when not yet set */
......
......@@ -18,7 +18,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Reference }
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction, Reference }
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
/**
......@@ -26,7 +26,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
*
* Created by pjvan_thof on 3/3/15.
*/
class Freebayes(val root: Configurable) extends BiopetCommandLineFunction with Reference {
class Freebayes(val root: Configurable) extends BiopetCommandLineFunction with Reference with Version {
@Input(required = true)
var bamfiles: List[File] = Nil
......@@ -41,8 +41,8 @@ class Freebayes(val root: Configurable) extends BiopetCommandLineFunction with R
var haplotypeLength: Option[Int] = config("haplotype_length")
executable = config("exe", default = "freebayes")
override def versionRegex = """version: (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """version: (.*)""".r
def versionCommand = executable + " --version"
override def beforeGraph(): Unit = {
super.beforeGraph()
......
......@@ -18,14 +18,14 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Reference }
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction, Reference }
import org.broadinstitute.gatk.utils.commandline.{ Argument, Input, Output }
/**
* Wrapper for the gsnap command line tool
* Written based on gsnap version 2014-05-15
*/
class Gsnap(val root: Configurable) extends BiopetCommandLineFunction with Reference {
class Gsnap(val root: Configurable) extends BiopetCommandLineFunction with Reference with Version {
/** default executable */
executable = config("exe", default = "gsnap", freeVar = false)
......@@ -328,8 +328,8 @@ class Gsnap(val root: Configurable) extends BiopetCommandLineFunction with Refer
/** value to put into read-group library (rg-pl) field */
var read_group_platform: Option[String] = config("read_group_platform")
override def versionRegex = """.* version (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """.* version (.*)""".r
def versionCommand = executable + " --version"
def cmdLine = {
required(executable) +
......
......@@ -17,11 +17,11 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
class Gzip(val root: Configurable) extends BiopetCommandLineFunction {
class Gzip(val root: Configurable) extends BiopetCommandLineFunction with Version {
@Input(doc = "Input file", required = true)
var input: List[File] = Nil
......@@ -30,8 +30,8 @@ class Gzip(val root: Configurable) extends BiopetCommandLineFunction {
executable = config("exe", default = "gzip", freeVar = false)
override def versionRegex = """gzip (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """gzip (.*)""".r
def versionCommand = executable + " --version"
def cmdLine = required(executable) + " -c " +
(if (inputAsStdin) "" else repeat(input)) +
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
* Wrapper for the htseq-count command line tool
* Written based on htseq-count version 0.6.1p1
*/
class HtseqCount(val root: Configurable) extends BiopetCommandLineFunction {
class HtseqCount(val root: Configurable) extends BiopetCommandLineFunction with Version {
/** default executable */
executable = config("exe", default = "htseq-count")
......@@ -70,8 +70,8 @@ class HtseqCount(val root: Configurable) extends BiopetCommandLineFunction {
/** suppress progress report */
var quiet: Boolean = config("quiet", default = false)
override def versionRegex = """.*, version (.*)\.""".r
override def versionCommand = executable + " --help"
def versionRegex = """.*, version (.*)\.""".r
def versionCommand = executable + " --help"
def cmdLine = {
required(executable) +
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{ Argument, Input, Output }
......@@ -27,11 +27,11 @@ import scalaz.std.boolean.option
* extension for raxml
* based on version 8.1.3
*/
class Raxml(val root: Configurable) extends BiopetCommandLineFunction {
class Raxml(val root: Configurable) extends BiopetCommandLineFunction with Version {
override def defaultThreads = 1
override def versionCommand = executable + " -v"
override def versionRegex = """.*version ([\w\.]*) .*""".r
def versionCommand = executable + " -v"
def versionRegex = """.*version ([\w\.]*) .*""".r
@Input(doc = "Input phy/fasta file", required = true)
var input: File = _
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
import nl.lumc.sasc.biopet.core.{ Version, BiopetCommandLineFunction }
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.summary.Summarizable
import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
......@@ -29,7 +29,7 @@ import scala.io.Source
* Extension for sickle
* Based on version 1.33
*/
class Sickle(val root: Configurable) extends BiopetCommandLineFunction with Summarizable {
class Sickle(val root: Configurable) extends BiopetCommandLineFunction with Summarizable with Version {
@Input(doc = "R1 input")
var input_R1: File = _
......@@ -56,8 +56,8 @@ class Sickle(val root: Configurable) extends BiopetCommandLineFunction with Summ
var discardN: Boolean = config("discardN", default = false)
var quiet: Boolean = config("quiet", default = false)
var defaultQualityType: String = config("defaultqualitytype", default = "sanger")
override def versionRegex = """sickle version (.*)""".r
override def versionCommand = executable + " --version"
def versionRegex = """sickle version (.*)""".r
def versionCommand = executable + " --version"
/** Sets qualityType is still empty */
override def beforeGraph() {
......
......@@ -18,11 +18,11 @@ package nl.lumc.sasc.biopet.extensions
import java.io.File