Commit bbc3592a authored by Sander Bollen's avatar Sander Bollen
Browse files

Merge branch 'feature-fixed_values' into 'develop'

Feature fixed values

See #126 

Order of prio of config values:
1. Fixed values
2. User config (-config / -cv on the command line)
3. Default files (coming from BIOPET_CONFIG env)
4. Default values inside the code.

Also changed way to use default values. No need for a merge step anymore, this happens inside the class now. This makes code inside pipelines/extensions cleaner.

Todo:
- [x] Basic implementation
- [x] Unit testing

See merge request !233
parents 1d3258b8 1a36a81e
......@@ -153,11 +153,11 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
scriptOutput.rawVcfFile = m2v.output
val vcfFilter = new VcfFilter(this) {
override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
override def defaults = Map("min_sample_depth" -> 8,
"min_alternate_depth" -> 2,
"min_samples_pass" -> 1,
"filter_ref_calls" -> true
), super.defaults)
)
}
vcfFilter.inputVcf = m2v.output
vcfFilter.outputVcf = swapExt(outputDir, m2v.output, ".vcf", ".filter.vcf.gz")
......
......@@ -35,10 +35,10 @@ trait BastyTrait extends MultiSampleQScript {
def variantcallers = List("freebayes")
override def defaults = ConfigUtils.mergeMaps(Map(
override def defaults = Map(
"ploidy" -> 1,
"variantcallers" -> variantcallers
), super.defaults)
)
lazy val shiva: ShivaTrait = new Shiva(qscript)
......
......@@ -47,7 +47,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Refe
override def defaultCoreMemory = 4.0
override def defaultThreads = 8
var sam: Boolean = config("sam", default = true)
var sam: Boolean = config("sam", default = false)
var sam_RG: Option[String] = config("sam-RG")
var seedlen: Option[Int] = config("seedlen")
var seedmms: Option[Int] = config("seedmms")
......
......@@ -25,17 +25,10 @@ import scala.sys.process.{ Process, ProcessLogger }
class GsnapTest extends TestNGSuite with Matchers {
private def setConfig(key: String, value: String): Map[String, Any] = {
val oldMap: Map[String, Any] = Config.global.map
Config.global.map += (key -> value)
oldMap
}
private def restoreConfig(oldMap: Map[String, Any]): Unit = Config.global.map = oldMap
@BeforeClass def checkExecutable() = {
val oldMap = setConfig("db", "mock")
val wrapper = new Gsnap(null)
val wrapper = new Gsnap(null) {
override def globalConfig = new Config(Map("db" -> "mock"))
}
val proc = Process(wrapper.versionCommand)
val exitCode =
try {
......@@ -47,13 +40,12 @@ class GsnapTest extends TestNGSuite with Matchers {
}
if (exitCode != 0)
throw new SkipException("Skipping GSNAP test because the executable can not be found")
restoreConfig(oldMap)
}
@Test(description = "GSNAP version number capture from executable")
def testVersion() = {
val oldMap = setConfig("db", "mock")
new Gsnap(null).getVersion should not be "N/A"
restoreConfig(oldMap)
new Gsnap(null) {
override def globalConfig = new Config(Map("db" -> "mock"))
}.getVersion should not be "N/A"
}
}
......@@ -21,11 +21,11 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils._
/**
* This class can store nested config values
* @param map Map with value for new config
* @param _map Map with value for new config
* @constructor Load config with existing map
*/
class Config(var map: Map[String, Any],
protected[config] var defaults: Map[String, Any] = Map()) extends Logging {
class Config(protected var _map: Map[String, Any],
protected var _defaults: Map[String, Any] = Map()) extends Logging {
logger.debug("Init phase of config")
/** Default constructor */
......@@ -34,6 +34,9 @@ class Config(var map: Map[String, Any],
loadDefaultConfig()
}
def map = _map
def defaults = _defaults
/**
* Loading a environmental variable as location of config files to merge into the config
* @param valueName Name of value
......@@ -65,13 +68,13 @@ class Config(var map: Map[String, Any],
def loadConfigFile(configFile: File, default: Boolean = false) {
val configMap = fileToConfigMap(configFile)
if (default) {
if (defaults.isEmpty) defaults = configMap
else defaults = mergeMaps(configMap, defaults)
logger.debug("New defaults: " + defaults)
if (_defaults.isEmpty) _defaults = configMap
else _defaults = mergeMaps(configMap, _defaults)
logger.debug("New defaults: " + _defaults)
} else {
if (map.isEmpty) map = configMap
else map = mergeMaps(configMap, map)
logger.debug("New config: " + map)
if (_map.isEmpty) _map = configMap
else _map = mergeMaps(configMap, _map)
logger.debug("New config: " + _map)
}
}
......@@ -84,11 +87,12 @@ class Config(var map: Map[String, Any],
*/
def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
if (default) defaults = mergeMaps(valueMap, defaults)
else map = mergeMaps(valueMap, map)
if (default) _defaults = mergeMaps(valueMap, _defaults)
else _map = mergeMaps(valueMap, _map)
}
protected[config] var notFoundCache: List[ConfigValueIndex] = List()
protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] def clearCache(): Unit = {
......@@ -103,24 +107,39 @@ class Config(var map: Map[String, Any],
* @param s key
* @return True if exist
*/
def contains(s: String): Boolean = map.contains(s)
def contains(s: String): Boolean = _map.contains(s)
/**
* Checks if value exist in config
* @param requestedIndex Index to value
* @return True if exist
*/
def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())
/**
* Checks if value exist in config
* @param requestedIndex Index to value
* @param fixedValues Fixed values
* @return True if exist
*/
def contains(requestedIndex: ConfigValueIndex): Boolean =
def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
if (notFoundCache.contains(requestedIndex)) false
else if (fixedCache.contains(requestedIndex)) true
else if (foundCache.contains(requestedIndex)) true
else {
val value = Config.getValueFromMap(map, requestedIndex)
if (value.isDefined && value.get.value != None) {
foundCache += (requestedIndex -> value.get)
val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
if (fixedValue.isDefined) {
fixedCache += (requestedIndex -> fixedValue.get)
true
} else {
notFoundCache +:= requestedIndex
false
val value = Config.getValueFromMap(_map, requestedIndex)
if (value.isDefined && value.get.value != None) {
foundCache += (requestedIndex -> value.get)
true
} else {
notFoundCache +:= requestedIndex
false
}
}
}
......@@ -132,9 +151,12 @@ class Config(var map: Map[String, Any],
* @param freeVar Default true, if set false value must exist in module
* @return True if exist
*/
def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
def contains(module: String, path: List[String],
key: String,
freeVar: Boolean = true,
fixedValues: Map[String, Any] = Map()): Boolean = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
contains(requestedIndex)
contains(requestedIndex, fixedValues)
}
/**
......@@ -146,10 +168,23 @@ class Config(var map: Map[String, Any],
* @param freeVar Default true, if set false value must exist in module
* @return Config value
*/
protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
protected[config] def apply(module: String,
path: List[String],
key: String,
default: Any = null,
freeVar: Boolean = true,
fixedValues: Map[String, Any] = Map()): ConfigValue = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
if (contains(requestedIndex)) foundCache(requestedIndex)
else if (default != null) {
if (contains(requestedIndex, fixedValues)) {
val fixedValue = fixedCache.get(requestedIndex)
if (fixedValue.isDefined) {
val userValue = Config.getValueFromMap(_map, requestedIndex)
if (userValue.isDefined)
logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
}
fixedValue.getOrElse(foundCache(requestedIndex))
} else if (default != null) {
defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
defaultCache(requestedIndex)
} else ConfigValue(requestedIndex, null, null, freeVar)
......@@ -179,9 +214,11 @@ class Config(var map: Map[String, Any],
// Positions where values are found
val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
// Positions where to start searching
val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val notFound = convertIndexValuesToMap(notFoundCache.map((_, None)), Some(false))
......@@ -189,16 +226,19 @@ class Config(var map: Map[String, Any],
val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)
writeMapToJsonFile(this.map, "input")
writeMapToJsonFile(_map, "input")
writeMapToJsonFile(_defaults, "defaults")
writeMapToJsonFile(found, "found")
writeMapToJsonFile(fixed, "fixed")
writeMapToJsonFile(effectiveFound, "effective.found")
writeMapToJsonFile(effectiveFixed, "effective.fixed")
writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
writeMapToJsonFile(notFound, "not.found")
writeMapToJsonFile(fullEffective, "effective.full")
writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
}
override def toString: String = map.toString()
override def toString: String = _map.toString()
}
object Config extends Logging {
......@@ -210,7 +250,7 @@ object Config extends Logging {
* @param config2 Low prio map
* @return Merged config
*/
def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
/**
* Search for value in index position in a map
......
......@@ -15,6 +15,7 @@
*/
package nl.lumc.sasc.biopet.utils.config
import nl.lumc.sasc.biopet.utils.ConfigUtils
import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
trait Configurable extends ImplicitConversions {
......@@ -35,9 +36,28 @@ trait Configurable extends ImplicitConversions {
def configFullPath: List[String] = configPath ::: configName :: Nil
/** Map to store defaults for config */
def defaults: Map[String, Any] = {
if (root != null) root.defaults
else globalConfig.defaults
def defaults: Map[String, Any] = Map()
/** This method merge defaults from the root to it's own */
protected def internalDefaults: Map[String, Any] = {
(root != null, defaults.isEmpty) match {
case (true, true) => root.defaults
case (true, false) => ConfigUtils.mergeMaps(defaults, root.defaults)
case (false, true) => globalConfig.defaults
case (false, false) => ConfigUtils.mergeMaps(defaults, globalConfig.defaults)
}
}
/** All values found in this map will be skipped from the user config */
def fixedValues: Map[String, Any] = Map()
/** This method merge fixedValues from the root to it's own */
protected def internalFixedValues: Map[String, Any] = {
(root != null, fixedValues.isEmpty) match {
case (true, true) => root.internalFixedValues
case (true, false) => ConfigUtils.mergeMaps(fixedValues, root.internalFixedValues)
case _ => fixedValues
}
}
val config = new ConfigFunctions
......@@ -90,11 +110,11 @@ trait Configurable extends ImplicitConversions {
val m = if (submodule != null) submodule else configName
val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
val d = {
val value = Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar))
val value = Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar))
if (value.isDefined) value.get.value else default
}
if (d == null) globalConfig(m, p, key, freeVar = freeVar)
else globalConfig(m, p, key, d, freeVar)
if (d == null) globalConfig(m, p, key, freeVar = freeVar, fixedValues = internalFixedValues)
else globalConfig(m, p, key, d, freeVar, fixedValues = internalFixedValues)
}
/**
......@@ -117,7 +137,7 @@ trait Configurable extends ImplicitConversions {
val m = if (submodule != null) submodule else configName
val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
globalConfig.contains(m, p, key, freeVar) || Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
globalConfig.contains(m, p, key, freeVar, internalFixedValues) || Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
}
}
}
......@@ -25,10 +25,36 @@ import org.testng.annotations.Test
* Created by pjvan_thof on 1/8/15.
*/
class ConfigurableTest extends TestNGSuite with Matchers {
abstract class Cfg extends Configurable {
def get(key: String,
default: String = null,
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null) = {
config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
}
}
class ClassA(val root: Configurable) extends Cfg
class ClassB(val root: Configurable) extends Cfg {
lazy val classA = new ClassA(this)
// Why this needs to be lazy?
}
class ClassC(val root: Configurable) extends Cfg {
def this() = this(null)
lazy val classB = new ClassB(this)
// Why this needs to be lazy?
}
@Test def testConfigurable(): Unit = {
val classC = new ClassC {
override def configName = "classc"
override val globalConfig = new Config(ConfigurableTest.map)
override val fixedValues = Map("fixed" -> "fixed")
}
classC.configPath shouldBe Nil
......@@ -51,46 +77,33 @@ class ConfigurableTest extends TestNGSuite with Matchers {
classC.get("bla", sample = "sample1", library = "library1").asString shouldBe "bla"
classC.get("test", sample = "sample1", library = "library1").asString shouldBe "test"
classC.get("test", sample = "sample1").asString shouldBe "test"
}
}
abstract class Cfg extends Configurable {
def get(key: String,
default: String = null,
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null) = {
config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
// Fixed values
classC.get("fixed").asString shouldBe "fixed"
classC.classB.get("fixed").asString shouldBe "fixed"
classC.classB.classA.get("fixed").asString shouldBe "fixed"
}
}
class ClassA(val root: Configurable) extends Cfg
class ClassB(val root: Configurable) extends Cfg {
lazy val classA = new ClassA(this)
// Why this needs to be lazy?
}
class ClassC(val root: Configurable) extends Cfg {
def this() = this(null)
lazy val classB = new ClassB(this)
// Why this needs to be lazy?
}
object ConfigurableTest {
val map = Map(
"fixed" -> "nonfixed",
"classa" -> Map(
"k1" -> "a1"
"k1" -> "a1",
"fixed" -> "nonfixed"
), "classb" -> Map(
"k1" -> "b1"
"k1" -> "b1",
"fixed" -> "nonfixed"
), "classc" -> Map(
"k1" -> "c1"
"k1" -> "c1",
"fixed" -> "nonfixed"
), "samples" -> Map(
"sample1" -> Map(
"fixed" -> "nonfixed",
"test" -> "test",
"libraries" -> Map(
"library1" -> Map(
"fixed" -> "nonfixed",
"bla" -> "bla"
)
)
......
......@@ -38,12 +38,12 @@ class Carp(val root: Configurable) extends QScript with MultiSampleQScript with
qscript =>
def this() = this(null)
override def defaults = ConfigUtils.mergeMaps(Map(
override def defaults = Map(
"mapping" -> Map(
"skip_markduplicates" -> true,
"aligner" -> "bwa-mem"
)
), super.defaults)
)
def summaryFile = new File(outputDir, "Carp.summary.json")
......
......@@ -100,24 +100,23 @@ class Gentrap(val root: Configurable) extends QScript
})
/** Default pipeline config */
override def defaults = ConfigUtils.mergeMaps(
Map(
"gsnap" -> Map(
"novelsplicing" -> 1,
"batch" -> 4,
"format" -> "sam"
),
"cutadapt" -> Map("minimum_length" -> 20),
// avoid conflicts when merging since the MarkDuplicate tags often cause merges to fail
"picard" -> Map(
"programrecordid" -> "null"
),
// disable markduplicates since it may not play well with all aligners (this can still be overriden via config)
"mapping" -> Map(
"skip_markduplicates" -> true,
"skip_metrics" -> true
)
), super.defaults)
override def defaults = Map(
"gsnap" -> Map(
"novelsplicing" -> 1,
"batch" -> 4,
"format" -> "sam"
),
"cutadapt" -> Map("minimum_length" -> 20),
// avoid conflicts when merging since the MarkDuplicate tags often cause merges to fail
"picard" -> Map(
"programrecordid" -> "null"
),
// disable markduplicates since it may not play well with all aligners (this can still be overriden via config)
"mapping" -> Map(
"skip_markduplicates" -> true,
"skip_metrics" -> true
)
)
/** Adds output merge jobs for the given expression mode */
// TODO: can we combine the enum with the file extension (to reduce duplication and potential errors)
......
......@@ -97,13 +97,12 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
/** location of summary file */
def summaryFile = new File(outputDir, sampleId.getOrElse("x") + "-" + libId.getOrElse("x") + ".summary.json")
override def defaults = ConfigUtils.mergeMaps(
Map(
"gsnap" -> Map(
"batch" -> 4,
"format" -> "sam"
)
), super.defaults)
override def defaults = Map("gsnap" -> Map("batch" -> 4))
override def fixedValues = Map(
"gsnap" -> Map("format" -> "sam"),
"bowtie" -> Map("sam" -> true)
)
/** File to add to the summary */
def summaryFiles: Map[String, File] = Map("output_bamfile" -> finalBamFile, "input_R1" -> input_R1,
......
......@@ -36,21 +36,22 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
var transcriptome: Option[File] = config("transcriptome")
var tagsLibrary: Option[File] = config("tags_library")
override def defaults = ConfigUtils.mergeMaps(Map("bowtie" -> Map(
"m" -> 1,
"k" -> 1,
"best" -> true,
"strata" -> true,
"seedmms" -> 1
), "mapping" -> Map(
"aligner" -> "bowtie",
"skip_flexiprep" -> true,
"skip_markduplicates" -> true
), "flexiprep" -> Map(
"skip_clip" -> true,
"skip_trim" -> true
), "strandSensitive" -> true
), super.defaults)
override def defaults = Map(
"bowtie" -> Map(
"m" -> 1,
"k" -> 1,
"best" -> true,
"strata" -> true,
"seedmms" -> 1
), "mapping" -> Map(
"aligner" -> "bowtie",
"skip_flexiprep" -> true,
"skip_markduplicates" -> true
), "flexiprep" -> Map(
"skip_clip" -> true,
"skip_trim" -> true
), "strandSensitive" -> true
)
def summaryFile: File = new File(outputDir, "Sage.summary.json")
......
......@@ -199,11 +199,11 @@ trait ShivaVariantcallingTrait extends SummaryQScript with SampleLibraryTag with
val vcfFilter = new VcfFilter(qscript) {
override def configName = "vcffilter"
override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
override def defaults = Map("min_sample_depth" -> 8,
"min_alternate_depth" -> 2,
"min_samples_pass" -> 1,
"filter_ref_calls" -> true
), super.defaults)
)
}
vcfFilter.inputVcf = m2v.output