Commit bd732f66 authored by bow's avatar bow
Browse files

Merge branch 'feature-config_unittest' into 'develop'

Feature config unittest

See also #89

Also some start for issue #55

See merge request !65
parents 86e3b9eb 43f8ced5
......@@ -62,7 +62,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
}
val multisampleVariantcalling = new GatkVariantcalling(this) {
override protected lazy val configName = "gatkvariantcalling"
override def configName = "gatkvariantcalling"
override def configPath: List[String] = "multisample" :: super.configPath
}
......@@ -97,7 +97,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
val allRawVcfFiles = for ((sampleID, sampleOutput) <- samplesOutput) yield sampleOutput.variantcalling.rawFilterVcfFile
val gatkVariantcalling = new GatkVariantcalling(this) {
override protected lazy val configName = "gatkvariantcalling"
override def configName = "gatkvariantcalling"
override def configPath: List[String] = "multisample" :: super.configPath
}
......
......@@ -15,7 +15,7 @@
*/
package nl.lumc.sasc.biopet.core
import nl.lumc.sasc.biopet.core.config.{ Config, Configurable }
import nl.lumc.sasc.biopet.core.config.{ ConfigValue, Config, Configurable }
import nl.lumc.sasc.biopet.utils.ConfigUtils._
trait MultiSampleQScript extends BiopetQScript {
......@@ -29,31 +29,61 @@ trait MultiSampleQScript extends BiopetQScript {
def getLibrary(key: String) = libraries(key)
}
var samplesConfig: Map[String, Any] = config("samples")
var samplesOutput: Map[String, SampleOutput] = Map()
if (!config.contains("samples")) logger.warn("No Samples found in config")
/**
* Returns a map with all sample configs
*/
val getSamplesConfig: Map[String, Any] = config("samples", default = Map())
/**
* Returns a list of all sampleIDs
*/
val getSamples: Set[String] = getSamplesConfig.keySet
/**
* Returns the global sample directory
* @return global sample directory
*/
def globalSampleDir: String = outputDir + "samples/"
var samplesOutput: Map[String, SampleOutput] = Map()
/**
* Runs runSingleSampleJobs method for each sample
*/
final def runSamplesJobs() {
if (samplesConfig == null) samplesConfig = Map()
if (Config.global.contains("samples")) for ((key, value) <- samplesConfig) {
for ((key, value) <- getSamplesConfig) {
var sample = any2map(value)
if (!sample.contains("ID")) sample += ("ID" -> key)
if (sample("ID") == key) {
currentSample = key
samplesOutput += key -> runSingleSampleJobs(sample)
currentSample = null
} else logger.warn("Key is not the same as ID on value for sample")
}
else logger.warn("No Samples found in config")
}
def runSingleSampleJobs(sampleConfig: Map[String, Any]): SampleOutput
/**
* Run sample with only sampleID
* @param sample sampleID
* @return
*/
def runSingleSampleJobs(sample: String): SampleOutput = {
var map = any2map(samplesConfig(sample))
var map = any2map(getSamplesConfig(sample))
if (map.contains("ID") && map("ID") != sample)
throw new IllegalStateException("ID in config not the same as the key")
else map += ("ID" -> sample)
return runSingleSampleJobs(map)
}
/**
* Runs runSingleLibraryJobs method for each library found in sampleConfig
* @param sampleConfig sample config
* @return Map with libraryID -> LibraryOutput object
*/
final def runLibraryJobs(sampleConfig: Map[String, Any]): Map[String, LibraryOutput] = {
var output: Map[String, LibraryOutput] = Map()
val sampleID = sampleConfig("ID").toString
......@@ -63,11 +93,88 @@ trait MultiSampleQScript extends BiopetQScript {
var library = any2map(value)
if (!library.contains("ID")) library += ("ID" -> key)
if (library("ID") == key) {
currentLibrary = key
output += key -> runSingleLibraryJobs(library, sampleConfig)
currentLibrary = null
} else logger.warn("Key is not the same as ID on value for run of sample: " + sampleID)
}
} else logger.warn("No runs found in config for sample: " + sampleID)
return output
}
def runSingleLibraryJobs(runConfig: Map[String, Any], sampleConfig: Map[String, Any]): LibraryOutput
protected var currentSample: String = null
protected var currentLibrary: String = null
/**
* Set current sample manual, only use this when not using runSamplesJobs method
* @param sample
*/
def setCurrentSample(sample: String) {
logger.debug("Manual sample set to: " + sample)
currentSample = sample
}
/**
* Gets current sample
* @return current sample
*/
def getCurrentSample = currentSample
/**
* Reset current sample manual, only use this when not using runSamplesJobs method
*/
def resetCurrentSample() {
logger.debug("Manual sample reset")
currentSample = null
}
/**
* Set current library manual, only use this when not using runLibraryJobs method
* @param library
*/
def setCurrentLibrary(library: String) {
logger.debug("Manual library set to: " + library)
currentLibrary = library
}
/**
* Gets current library
* @return current library
*/
def getCurrentLibrary = currentLibrary
/**
* Reset current library manual, only use this when not using runLibraryJobs method
*/
def resetCurrentLibrary() {
logger.debug("Manual library reset")
currentLibrary = null
}
override protected[core] def configFullPath: List[String] = {
(if (currentSample != null) "samples" :: currentSample :: Nil else Nil) :::
(if (currentLibrary != null) "libraries" :: currentLibrary :: Nil else Nil) :::
super.configFullPath
}
protected class ConfigFunctions extends super.ConfigFunctions {
override def apply(key: String,
default: Any = null,
submodule: String = null,
required: Boolean = false,
freeVar: Boolean = true,
sample: String = currentSample,
library: String = currentLibrary): ConfigValue = {
super.apply(key, default, submodule, required, freeVar, sample, library)
}
override def contains(key: String,
submodule: String = null,
freeVar: Boolean = true,
sample: String = currentSample,
library: String = currentLibrary) = {
super.contains(key, submodule, freeVar, sample, library)
}
}
}
......@@ -16,10 +16,10 @@
package nl.lumc.sasc.biopet.core
trait ToolCommand extends MainCommand with Logging {
abstract class AbstractArgs {
protected abstract class AbstractArgs {
}
abstract class AbstractOptParser extends scopt.OptionParser[Args](commandName) {
protected abstract class AbstractOptParser extends scopt.OptionParser[Args](commandName) {
opt[String]('l', "log_level") foreach { x =>
x.toLowerCase match {
case "debug" => logger.setLevel(org.apache.log4j.Level.DEBUG)
......@@ -44,6 +44,6 @@ trait ToolCommand extends MainCommand with Logging {
} text ("Print version")
}
type Args <: AbstractArgs
type OptParser <: AbstractOptParser
protected type Args <: AbstractArgs
protected type OptParser <: AbstractOptParser
}
\ No newline at end of file
......@@ -19,18 +19,31 @@ import java.io.File
import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.utils.ConfigUtils._
/**
* This class can store nested config values
* @param map Map with value for new config
* @constructor Load config with existing map
*/
class Config(var map: Map[String, Any]) extends Logging {
logger.debug("Init phase of config")
/**
* Default constructor
*/
def this() = {
this(Map())
loadDefaultConfig()
}
/**
* Loading a environmental variable as location of config files to merge into the config
* @param valueName Name of value
*/
def loadConfigEnv(valueName: String) {
val globalFiles = sys.env.get(valueName).getOrElse("").split(":")
if (globalFiles.isEmpty) logger.info(valueName + " value not found, no global config is loaded")
for (globalFile <- globalFiles) {
var file: File = new File(globalFile)
val file: File = new File(globalFile)
if (file.exists()) {
logger.info("Loading config file: " + file)
loadConfigFile(file)
......@@ -38,10 +51,17 @@ class Config(var map: Map[String, Any]) extends Logging {
}
}
/**
* Loading default value for biopet
*/
def loadDefaultConfig() {
loadConfigEnv("BIOPET_CONFIG")
}
/**
* Merge a json file into the config
* @param configFile Location of file
*/
def loadConfigFile(configFile: File) {
val configMap = fileToConfigMap(configFile)
......@@ -53,12 +73,26 @@ class Config(var map: Map[String, Any]) extends Logging {
protected[config] var notFoundCache: List[ConfigValueIndex] = List()
protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] def clearCache: Unit = {
notFoundCache = List()
foundCache = Map()
defaultCache = Map()
}
/**
* Check if value exist in root of config
* @deprecated
* @param s key
* @return True if exist
*/
def contains(s: String): Boolean = map.contains(s)
def contains(requestedIndex: ConfigValueIndex, freeVar: Boolean): Boolean = contains(requestedIndex.module, requestedIndex.path, requestedIndex.key, freeVar)
def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex.module, requestedIndex.path, requestedIndex.key, true)
def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
/**
* Checks if value exist in config
* @param requestedIndex Index to value
* @return True if exist
*/
def contains(requestedIndex: ConfigValueIndex): Boolean =
if (notFoundCache.contains(requestedIndex)) return false
else if (foundCache.contains(requestedIndex)) return true
else {
......@@ -71,22 +105,45 @@ class Config(var map: Map[String, Any]) extends Logging {
return false
}
}
/**
* Checks if value exist in config
* @param module Name of module
* @param path Path to start searching
* @param key Name of value
* @param freeVar Default true, if set false value must exist in module
* @return True if exist
*/
def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
contains(requestedIndex)
}
/**
* Find value in config
* @param module Name of module
* @param path Path to start searching
* @param key Name of value
* @param default Default value when no value is found
* @param freeVar Default true, if set false value must exist in module
* @return Config value
*/
protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
val requestedIndex = ConfigValueIndex(module, path, key)
if (contains(requestedIndex, freeVar)) return foundCache(requestedIndex)
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
if (contains(requestedIndex)) return foundCache(requestedIndex)
else if (default != null) {
defaultCache += (requestedIndex -> ConfigValue.apply(requestedIndex, null, default, true))
defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
return defaultCache(requestedIndex)
} else {
logger.error("Value in config could not be found but it seems required, index: " + requestedIndex)
throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
}
} else throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
}
//TODO: New version of report is needed
/**
* Makes report for all used values
* @return Config report
*/
def getReport: String = {
var output: StringBuilder = new StringBuilder
val output: StringBuilder = new StringBuilder
output.append("Config report, sorted on module:\n")
var modules: Map[String, StringBuilder] = Map()
for ((key, value) <- foundCache) {
......@@ -118,43 +175,60 @@ class Config(var map: Map[String, Any]) extends Logging {
object Config extends Logging {
val global = new Config
/**
* Merge 2 config objects
* @param config1 prio over config 2
* @param config2
* @return Merged config
*/
def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
private def getMapFromPath(map: Map[String, Any], path: List[String]): Map[String, Any] = {
var returnMap: Map[String, Any] = map
for (m <- path) {
if (!returnMap.contains(m)) return Map()
else returnMap = any2map(returnMap(m))
/**
* Search for value in index position in a map
* @param map Map to search in
* @param startIndex Config index
* @return Value
*/
def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
def getFromPath(path: List[String]): Option[ConfigValue] = {
val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
else if (startIndex.freeVar) {
val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
else None
} else None
}
return returnMap
}
def getValueFromMap(map: Map[String, Any], index: ConfigValueIndex): Option[ConfigValue] = {
var submodules = index.path.reverse
while (!submodules.isEmpty) {
var submodules2 = submodules
while (!submodules2.isEmpty) {
val p = getMapFromPath(map, submodules2 ::: index.module :: Nil)
if (p.contains(index.key)) {
return Option(ConfigValue(index, ConfigValueIndex(index.module, submodules2, index.key), p(index.key)))
}
if (index.freeVar) {
val p2 = getMapFromPath(map, submodules2)
if (p2.contains(index.key)) {
return Option(ConfigValue(index, ConfigValueIndex(index.module, submodules2, index.key), p2(index.key)))
}
}
submodules2 = submodules2.init
def tailSearch(path: List[String]): Option[ConfigValue] = {
val p = getFromPath(path)
if (p != None) p
else if (path == Nil) None
else {
val p = initSearch(path)
if (p.isDefined) p
else tailSearch(path.tail)
}
submodules = submodules.tail
}
val p = getMapFromPath(map, index.module :: Nil)
if (p.contains(index.key)) { // Module is not nested
return Option(ConfigValue(index, ConfigValueIndex(index.module, Nil, index.key), p(index.key)))
} else if (map.contains(index.key) && index.freeVar) { // Root value of json
return Option(ConfigValue(index, ConfigValueIndex("", Nil, index.key), map(index.key)))
} else { // At this point key is not found on the path
return None
def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
val p = getFromPath(path)
if (p.isDefined) p
else if (path == Nil) None
else {
val p = skipNested(path, tail)
if (p.isDefined) p
else initSearch(path.init, path.last :: tail)
}
}
def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
val p = getFromPath(path ::: tail)
if (p.isDefined) p
else if (tail == Nil) None
else skipNested(path, tail.tail)
}
return tailSearch(startIndex.path)
}
}
\ No newline at end of file
......@@ -19,15 +19,58 @@ import java.io.File
import nl.lumc.sasc.biopet.utils.ConfigUtils._
class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValueIndex, val value: Any, val default: Boolean) {
/**
* Get value as String
* @return value as String
*/
def asString = any2string(value)
/**
* Get value as Int
* @return value as Int
*/
def asInt = any2int(value)
/**
* Get value as Double
* @return value as Double
*/
def asDouble = any2double(value)
/**
* Get value as List[Any]
* @return value as List[Any]
*/
def asList = any2list(value)
/**
* Get value as List[File]
* @return value as List[File]
*/
def asFileList: List[File] = for (file <- any2stringList(value)) yield new File(file)
/**
* Get value as List[String]
* @return value as List[String]
*/
def asStringList: List[String] = any2stringList(value)
/**
* Get value as Map
* @return value as Map
*/
def asMap = any2map(value)
/**
* Get value as Boolean
* @return value as Boolean
*/
def asBoolean = any2boolean(value)
/**
* Readable output of indexes and value, just for debug
* @return
*/
override def toString: String = {
var output = "key = " + requestIndex.key
output += ", value = " + value
......@@ -41,9 +84,25 @@ class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValu
}
object ConfigValue {
/**
*
* @param requestIndex Index where to start searching
* @param foundIndex Index where value is found
* @param value Found value
* @return ConfigValue object
*/
def apply(requestIndex: ConfigValueIndex, foundIndex: ConfigValueIndex, value: Any) = {
new ConfigValue(requestIndex, foundIndex, value, false)
}
/**
*
* @param requestIndex Index where to start searching
* @param foundIndex Index where value is found
* @param value Found value
* @param default Value is a default value
* @return ConfigValue object
*/
def apply(requestIndex: ConfigValueIndex, foundIndex: ConfigValueIndex, value: Any, default: Boolean) = {
new ConfigValue(requestIndex, foundIndex, value, default)
}
......
......@@ -15,15 +15,13 @@
*/
package nl.lumc.sasc.biopet.core.config
class ConfigValueIndex(val module: String, val path: List[String], val key: String, val freeVar: Boolean = true) {
/**
* General case class used as index config values. This stores the path to the value, the module, name of the value and if freeVar is allowed
* @param module Module where this value is belonging to
* @param path Path to value
* @param key Name of value
* @param freeVar Default true, if set false value must exist in module
*/
case class ConfigValueIndex(module: String, path: List[String], key: String, freeVar: Boolean = true) {
override def toString = "Module = " + module + ", path = " + path + ", key = " + key + ", freeVar = " + freeVar
}
object ConfigValueIndex {
private var cache: Map[(String, List[String], String), ConfigValueIndex] = Map()
def apply(module: String, path: List[String], key: String, freeVar: Boolean = true): ConfigValueIndex = {
if (!cache.contains(module, path, key)) cache += ((module, path, key) -> new ConfigValueIndex(module, path, key, freeVar))
return cache(module, path, key)
}
}
\ No newline at end of file
......@@ -20,36 +20,105 @@ import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
trait Configurable extends ImplicitConversions {
/**
* Should be object of parant object
*/
val root: Configurable
/**
* Get default path to search config values for current object
* @return
*/
def configPath: List[String] = if (root != null) root.configFullPath else List()
protected lazy val configName = getClass.getSimpleName.toLowerCase
protected lazy val configFullPath = configName :: configPath
/**
* Gets name of module for config
* @return
*/
protected[core] def configName = getClass.getSimpleName.toLowerCase