Commit 9e88effb authored by bow's avatar bow
Browse files

Merge branch 'feature-fix_missing_values' into 'develop'

Feature fix missing values

Fix for wrong messages when config value does not exist

Now also fix for #112

See merge request !89
parents 1047c24a fbf67f2c
......@@ -13,13 +13,13 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction {
override def subPath = "gatk" :: super.subPath
jarFile = config("gatk_jar", required = true)
jarFile = config("gatk_jar")
override val defaultVmem = "7G"
if (config.contains("intervals")) intervals = config("intervals").asFileList
if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
reference_sequence = config("reference", required = true)
reference_sequence = config("reference")
if (config.contains("gatk_key")) gatk_key = config("gatk_key")
if (config.contains("pedigree")) pedigree = config("pedigree").asFileList
}
......@@ -35,7 +35,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
var jointGenotyping: Boolean = config("joint_genotyping", default = false)
var singleSampleCalling = config("single_sample_calling", default = true)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
var useAllelesOption: Boolean = config("use_alleles_option", default = false)
val externalGvcfs = config("external_gvcfs_files", default = Nil).asFileList
......
......@@ -30,7 +30,7 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
var rawVcfInput: File = _
@Argument(doc = "Reference", shortName = "R", required = false)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Argument(doc = "OutputName", required = false)
var outputName: String = _
......
......@@ -66,7 +66,7 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
super.freezeFieldValues()
}
protected def checkExecutable {
protected[core] def checkExecutable {
if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
try if (executable != null) {
if (!BiopetCommandLineFunctionTrait.executableCache.contains(executable)) {
......@@ -79,8 +79,7 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
val file = new File(executable)
executable = file.getCanonicalPath
} else {
logger.error("executable: '" + executable + "' not found, please check config")
throw new QException("executable: '" + executable + "' not found, please check config")
BiopetQScript.addError("executable: '" + executable + "' not found, please check config")
}
BiopetCommandLineFunctionTrait.executableCache += oldExecutable -> executable
BiopetCommandLineFunctionTrait.executableCache += executable -> executable
......@@ -101,9 +100,8 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
}
}
val md5 = BiopetCommandLineFunctionTrait.executableMd5Cache(executable)
if (md5 == null) addJobReportBinding("md5sum_exe", md5)
else addJobReportBinding("md5sum_exe", "None")
val md5 = BiopetCommandLineFunctionTrait.executableMd5Cache.get(executable)
addJobReportBinding("md5sum_exe", md5.getOrElse("None"))
}
final protected def preCmdInternal {
......@@ -120,6 +118,8 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
protected val versionExitcode = List(0) // Can select multiple
private def getVersionInternal: String = {
if (versionCommand == null || versionRegex == null) return "N/A"
val exe = new File(versionCommand.trim.split(" ")(0))
if (!exe.exists()) return "N/A"
val stdout = new StringBuffer()
val stderr = new StringBuffer()
def outputLog = "Version command: \n" + versionCommand +
......
......@@ -24,14 +24,16 @@ import org.broadinstitute.gatk.queue.function.QFunction
import org.broadinstitute.gatk.queue.function.scattergather.ScatterGatherableFunction
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
import scala.collection.mutable.ListBuffer
trait BiopetQScript extends Configurable with GatkLogging {
@Argument(doc = "JSON config file(s)", fullName = "config_file", shortName = "config", required = false)
val configfiles: List[File] = Nil
var outputDir: String = {
val temp = Config.getValueFromMap(Config.global.map, ConfigValueIndex(this.configName, configPath, "output_dir"))
if (temp.isEmpty) throw new IllegalArgumentException("No output_dir defined in config")
val temp = Config.getValueFromMap(globalConfig.map, ConfigValueIndex(this.configName, configPath, "output_dir"))
if (temp.isEmpty) ""
else {
val t = temp.get.value.toString
if (!t.endsWith("/")) t + "/" else t
......@@ -51,8 +53,9 @@ trait BiopetQScript extends Configurable with GatkLogging {
var functions: Seq[QFunction]
final def script() {
outputDir = config("output_dir", required = true)
if (!outputDir.endsWith("/")) outputDir += "/"
outputDir = config("output_dir")
if (outputDir.isEmpty) outputDir = new File(".").getAbsolutePath()
else if (!outputDir.endsWith("/")) outputDir += "/"
init
biopetScript
......@@ -61,11 +64,17 @@ trait BiopetQScript extends Configurable with GatkLogging {
case _ =>
}
for (function <- functions) function match {
case f: BiopetCommandLineFunctionTrait => f.afterGraph
case _ =>
case f: BiopetCommandLineFunctionTrait => {
f.checkExecutable
f.afterGraph
}
case _ =>
}
Config.global.writeReport(qSettings.runName, outputDir + ".log/" + qSettings.runName)
if (new File(outputDir).canWrite) globalConfig.writeReport(qSettings.runName, outputDir + ".log/" + qSettings.runName)
else BiopetQScript.addError("Output dir: '" + outputDir + "' is not writeable")
BiopetQScript.checkErrors
}
def add(functions: QFunction*) // Gets implemeted at org.broadinstitute.sting.queue.QScript
......@@ -73,5 +82,29 @@ trait BiopetQScript extends Configurable with GatkLogging {
function.isIntermediate = isIntermediate
add(function)
}
}
object BiopetQScript extends Logging {
private val errors: ListBuffer[Exception] = ListBuffer()
def addError(error: String, debug: String = null): Unit = {
val msg = error + (if (debug != null && logger.isDebugEnabled) "; " + debug else "")
errors.append(new Exception(msg))
}
protected def checkErrors: Unit = {
if (!errors.isEmpty) {
logger.error("*************************")
logger.error("Biopet found some errors:")
if (logger.isDebugEnabled) {
for (e <- errors) {
logger.error(e.getMessage)
logger.debug(e.getStackTrace.mkString("Stack trace:\n", "\n", "\n"))
}
} else {
errors.map(_.getMessage).sorted.distinct.foreach(logger.error(_))
}
throw new IllegalStateException("Biopet found errors")
}
}
}
\ No newline at end of file
......@@ -28,7 +28,7 @@ trait MultiSampleQScript extends BiopetQScript {
@Argument(doc = "Only Sample", shortName = "sample", required = false)
private val onlySamples: List[String] = Nil
require(Config.global.map.contains("samples"), "No Samples found in config")
require(globalConfig.map.contains("samples"), "No Samples found in config")
/**
* Sample class with basic functions build in
......@@ -80,7 +80,7 @@ trait MultiSampleQScript extends BiopetQScript {
/** returns a set with library names */
protected def libIds: Set[String] = {
ConfigUtils.getMapFromPath(Config.global.map, List("samples", sampleId, "libraries")).getOrElse(Map()).keySet
ConfigUtils.getMapFromPath(globalConfig.map, List("samples", sampleId, "libraries")).getOrElse(Map()).keySet
}
/** Adds sample jobs */
......@@ -125,7 +125,7 @@ trait MultiSampleQScript extends BiopetQScript {
val samples: Map[String, Sample] = sampleIds.map(id => id -> makeSample(id)).toMap
/** Returns a list of all sampleIDs */
protected def sampleIds: Set[String] = ConfigUtils.any2map(Config.global.map("samples")).keySet
protected def sampleIds: Set[String] = ConfigUtils.any2map(globalConfig.map("samples")).keySet
/** Runs addAndTrackJobs method for each sample */
final def addSamplesJobs() {
......
......@@ -133,11 +133,11 @@ class Config(var map: Map[String, Any]) extends Logging {
*/
protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
if (contains(requestedIndex)) return foundCache(requestedIndex)
if (contains(requestedIndex)) foundCache(requestedIndex)
else if (default != null) {
defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
return defaultCache(requestedIndex)
} else throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
defaultCache(requestedIndex)
} else ConfigValue(requestedIndex, null, null, freeVar)
}
def writeReport(id: String, directory: String): Unit = {
......@@ -174,7 +174,7 @@ class Config(var map: Map[String, Any]) extends Logging {
val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)
writeMapToJsonFile(Config.global.map, "input")
writeMapToJsonFile(this.map, "input")
writeMapToJsonFile(found, "found")
writeMapToJsonFile(effectiveFound, "effective.found")
writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
......
......@@ -21,6 +21,7 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
trait Configurable extends ImplicitConversions {
/** Should be object of parant object */
val root: Configurable
val globalConfig: Config = if (root != null) root.globalConfig else Config.global
/** subfix to the path */
def subPath: List[String] = Nil
......@@ -79,7 +80,6 @@ trait Configurable extends ImplicitConversions {
* @param key Name of value
* @param default Default value if not found
* @param submodule Adds to the path
* @param required Default false, if true and value is not found this function will raise an exception
* @param freeVar Default true, if set false value must exist in module
* @param sample Default null, when set path is prefixed with "samples" -> "sampleID"
* @param library Default null, when set path is prefixed with "libraries" -> "libraryID"
......@@ -88,7 +88,6 @@ trait Configurable extends ImplicitConversions {
def apply(key: String,
default: Any = null,
submodule: String = null,
required: Boolean = false,
freeVar: Boolean = true,
sample: String = null,
library: String = null): ConfigValue = {
......@@ -100,14 +99,8 @@ trait Configurable extends ImplicitConversions {
val value = Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar))
if (value.isDefined) value.get.value else default
}
if (!contains(key, submodule, freeVar, sample = s, library = l) && d == null) {
if (required) {
Logging.logger.error("Value in config could not be found but it is required, key: " + key + " module: " + m + " path: " + p)
throw new IllegalStateException("Value in config could not be found but it is required, key: " + key + " module: " + m + " path: " + p)
} else return null
}
if (d == null) return Config.global(m, p, key, freeVar = freeVar)
else return Config.global(m, p, key, d, freeVar)
if (d == null) globalConfig(m, p, key, freeVar = freeVar)
else globalConfig(m, p, key, d, freeVar)
}
/**
......@@ -129,7 +122,7 @@ trait Configurable extends ImplicitConversions {
val m = if (submodule != null) submodule else configName
val p = path(s, l, submodule)
Config.global.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
globalConfig.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
}
}
}
......@@ -81,19 +81,15 @@ object BiopetQCommandLine extends GatkLogging {
Runtime.getRuntime.addShutdownHook(shutdownHook)
CommandLineProgram.start(qCommandLine, argv)
try {
CommandLineProgram.start(qCommandLine, argv)
try {
Runtime.getRuntime.removeShutdownHook(shutdownHook)
qCommandLine.shutdown()
} catch {
case e: Exception => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
}
if (CommandLineProgram.result != 0)
System.exit(CommandLineProgram.result)
Runtime.getRuntime.removeShutdownHook(shutdownHook)
qCommandLine.shutdown()
} catch {
case e: Exception => CommandLineProgram.exitSystemWithError(e)
case e: Exception => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */
}
if (CommandLineProgram.result != 0)
System.exit(CommandLineProgram.result)
}
}
......@@ -114,21 +110,28 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
private var qScriptClasses: File = _
private var shuttingDown = false
/**
* we modified this in Biopet to skip compiling and show full stacktrace again
*/
private lazy val qScriptPluginManager = {
qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory)
//qScriptManager.loadScripts(scripts, qScriptClasses)
//var temp: Seq[URL] = Seq()
for (t <- scripts) {
val is = getClass.getResourceAsStream(t.getAbsolutePath)
val os = new FileOutputStream(qScriptClasses.getAbsolutePath + "/" + t.getName)
org.apache.commons.io.IOUtils.copy(is, os)
os.close()
//temp :+= this.getClass.getResource(t.toString)
//logger.info(this.getClass.getResource(t.toString))
val s = if (t.getName.endsWith("/")) t.getName.substring(0, t.getName.length - 1) else t.getName
pipelineName = s.substring(0, s.lastIndexOf(".")) + "." + System.currentTimeMillis
}
new PluginManager[QScript](qPluginType, List(qScriptClasses.toURI.toURL))
// override createByType to pass the correct exceptions
new PluginManager[QScript](qPluginType, List(qScriptClasses.toURI.toURL)) {
override def createByType(plugintype: Class[_ <: QScript]) = {
val noArgsConstructor = plugintype.getDeclaredConstructor()
noArgsConstructor.setAccessible(true)
noArgsConstructor.newInstance()
}
}
}
private lazy val qCommandPlugin = {
......@@ -186,12 +189,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
//if (settings.run)
script.pullInputs()
script.qSettings = settings.qSettings
try {
script.script()
} catch {
case e: Exception =>
throw new UserException.CannotExecuteQScript(script.getClass.getSimpleName + ".script() threw the following exception: " + e, e)
}
script.script()
if (remoteFileConverter != null) {
if (remoteFileConverter.convertToRemoteEnabled)
......
......@@ -29,7 +29,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction {
var R2: File = _
@Input(doc = "The reference file for the bam files.", shortName = "R")
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Output(doc = "Output file SAM", shortName = "output")
var output: File = _
......
......@@ -51,7 +51,7 @@ class Fastqc(val root: Configurable) extends BiopetCommandLineFunction {
override def afterGraph {
this.checkExecutable
val fastqcDir = executable.substring(0, executable.lastIndexOf("/"))
val fastqcDir = new File(executable).getParent
contaminants = contaminants match {
// user-defined contaminants file take precedence
......
......@@ -29,13 +29,13 @@ class Stampy(val root: Configurable) extends BiopetCommandLineFunction {
var R2: File = _
@Input(doc = "The reference file for the bam files.", shortName = "ref")
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Input(doc = "The genome prefix.")
var genome: File = config("genome", required = true)
var genome: File = config("genome")
@Input(doc = "The hash prefix")
var hash: File = config("hash", required = true)
var hash: File = config("hash")
@Output(doc = "Output file SAM", shortName = "output")
var output: File = _
......
......@@ -29,7 +29,7 @@ class TopHat(val root: Configurable) extends BiopetCommandLineFunction {
var R2: File = _
@Input(doc = "Bowtie index", shortName = "bti")
var bowtie_index: File = config("bowtie_index", required = true)
var bowtie_index: File = config("bowtie_index")
@Argument(doc = "Output Directory")
var outputDir: String = _
......
......@@ -13,7 +13,7 @@ class BwaAln(val root: Configurable) extends Bwa {
var fastq: File = _
@Input(doc = "The reference file for the bam files.", required = true)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Output(doc = "Output file SAM", required = false)
var output: File = _
......
......@@ -29,7 +29,7 @@ class BwaMem(val root: Configurable) extends Bwa {
var R2: File = _
@Input(doc = "The reference file for the bam files.", shortName = "R")
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Output(doc = "Output file SAM", shortName = "output")
var output: File = _
......
......@@ -26,7 +26,7 @@ class BwaSampe(val root: Configurable) extends Bwa {
var saiR2: File = _
@Input(doc = "The reference file for the bam files.", required = true)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Output(doc = "Output file SAM", required = false)
var output: File = _
......
......@@ -16,7 +16,7 @@ class BwaSamse(val root: Configurable) extends Bwa {
var sai: File = _
@Input(doc = "The reference file for the bam files.", required = true)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Output(doc = "Output file SAM", required = false)
var output: File = _
......
......@@ -35,7 +35,7 @@ class CollectGcBiasMetrics(val root: Configurable) extends Picard {
var outputSummary: File = _
@Argument(doc = "Reference file", required = false)
var reference: File = config("reference", required = true)
var reference: File = config("reference")
@Argument(doc = "Window size", required = false)
var windowSize: Option[Int] = config("windowsize")
......
......@@ -16,6 +16,7 @@
package nl.lumc.sasc.biopet.utils
import java.io.File
import nl.lumc.sasc.biopet.core.BiopetQScript
import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.core.config.ConfigValue
import argonaut._, Argonaut._
......@@ -330,14 +331,27 @@ object ConfigUtils extends Logging {
trait ImplicitConversions {
import scala.language.implicitConversions
private def requiredValue(value: ConfigValue): Boolean = {
val exist = valueExists(value)
if (!exist)
BiopetQScript.addError("Value does not exist but is required, key: " + value.requestIndex.key +
" module: " + value.requestIndex.module,
(if (value.requestIndex.path != Nil) " path: " + value.requestIndex.path.mkString("->") else null))
exist
}
private def valueExists(value: ConfigValue): Boolean = {
value != null && value.value != null && value.value != None
}
/**
* Convert ConfigValue to File
* @param value Input ConfigValue
* @return
*/
implicit def configValue2file(value: ConfigValue): File = {
if (value != null && value.value != null && value.value != None) new File(any2string(value.value))
else throw new IllegalStateException("Value does not exist")
if (requiredValue(value)) new File(any2string(value.value))
else new File("")
}
/**
......@@ -346,7 +360,7 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2optionFile(value: ConfigValue): Option[File] = {
if (value != null && value.value != null && value.value != None) Some(new File(any2string(value.value)))
if (valueExists(value)) Some(new File(any2string(value.value)))
else None
}
......@@ -356,8 +370,8 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2string(value: ConfigValue): String = {
if (value != null && value.value != null && value.value != None) any2string(value.value)
else throw new IllegalStateException("Value does not exist")
if (requiredValue(value)) any2string(value.value)
else ""
}
/**
......@@ -366,7 +380,7 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2optionString(value: ConfigValue): Option[String] = {
if (value != null && value.value != null && value.value != None) Some(any2string(value.value))
if (valueExists(value)) Some(any2string(value.value))
else None
}
......@@ -376,8 +390,8 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2long(value: ConfigValue): Long = {
if (value != null && value.value != null && value.value != None) any2long(value.value)
else throw new IllegalStateException("Value does not exist")
if (requiredValue(value)) any2long(value.value)
else 0L
}
/**
......@@ -386,7 +400,7 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2optionLong(value: ConfigValue): Option[Long] = {
if (value != null && value.value != null && value.value != None) Option(any2long(value.value))
if (valueExists(value)) Option(any2long(value.value))
else None
}
......@@ -396,8 +410,8 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2int(value: ConfigValue): Int = {
if (value != null && value.value != null && value.value != None) any2int(value.value)
else throw new IllegalStateException("Value does not exist")
if (requiredValue(value)) any2int(value.value)
else 0
}
/**
......@@ -406,7 +420,7 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2optionInt(value: ConfigValue): Option[Int] = {
if (value != null && value.value != null && value.value != None) Option(any2int(value.value))
if (valueExists(value)) Option(any2int(value.value))
else None
}
......@@ -416,8 +430,8 @@ object ConfigUtils extends Logging {
* @return
*/
implicit def configValue2double(value: ConfigValue): Double = {
if (value != null && value.value != null && value.value != None) any2double(value.value)
else throw new IllegalStateException("Value does not exist")
if (requiredValue(value)) any2double(value.value)
else 0.0
}
/**
......@@ -426,7 +440,7 @@ object ConfigUtils extends Logging {