Merge branch 'feature-fixed_values' into 'develop'

Feature fixed values

See #126 

Order of prio of config values:
1. Fixed values
2. User config (-config / -cv on the command line)
3. Default files (coming from BIOPET_CONFIG env)
4. Default values inside the code.

Also changed way to use default values. No need for a merge step anymore, this happens inside the class now. This makes code inside pipelines/extensions cleaner.

Todo:
- [x] Basic implementation
- [x] Unit testing

See merge request !233

protected/biopet-gatk-pipelines/src/main/scala/nl/lumc/sasc/biopet/pipelines/gatk/GatkVariantcalling.scala

@@ -153,11 +153,11 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
           scriptOutput.rawVcfFile = m2v.output
 
           val vcfFilter = new VcfFilter(this) {
-            override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
+            override def defaults = Map("min_sample_depth" -> 8,
               "min_alternate_depth" -> 2,
               "min_samples_pass" -> 1,
               "filter_ref_calls" -> true
-            ), super.defaults)
+            )
           }
           vcfFilter.inputVcf = m2v.output
           vcfFilter.outputVcf = swapExt(outputDir, m2v.output, ".vcf", ".filter.vcf.gz")

public/basty/src/main/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTrait.scala

@@ -35,10 +35,10 @@ trait BastyTrait extends MultiSampleQScript {
 
   def variantcallers = List("freebayes")
 
-  override def defaults = ConfigUtils.mergeMaps(Map(
+  override def defaults = Map(
     "ploidy" -> 1,
     "variantcallers" -> variantcallers
-  ), super.defaults)
+  )
 
   lazy val shiva: ShivaTrait = new Shiva(qscript)
 

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Bowtie.scala

@@ -47,7 +47,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Refe
   override def defaultCoreMemory = 4.0
   override def defaultThreads = 8
 
-  var sam: Boolean = config("sam", default = true)
+  var sam: Boolean = config("sam", default = false)
   var sam_RG: Option[String] = config("sam-RG")
   var seedlen: Option[Int] = config("seedlen")
   var seedmms: Option[Int] = config("seedmms")

public/biopet-extensions/src/test/scala/nl/lumc/sasc/biopet/extensions/GsnapTest.scala

@@ -25,17 +25,10 @@ import scala.sys.process.{ Process, ProcessLogger }
 
 class GsnapTest extends TestNGSuite with Matchers {
 
-  private def setConfig(key: String, value: String): Map[String, Any] = {
-    val oldMap: Map[String, Any] = Config.global.map
-    Config.global.map += (key -> value)
-    oldMap
-  }
-
-  private def restoreConfig(oldMap: Map[String, Any]): Unit = Config.global.map = oldMap
-
   @BeforeClass def checkExecutable() = {
-    val oldMap = setConfig("db", "mock")
-    val wrapper = new Gsnap(null)
+    val wrapper = new Gsnap(null) {
+      override def globalConfig = new Config(Map("db" -> "mock"))
+    }
     val proc = Process(wrapper.versionCommand)
     val exitCode =
       try {
@@ -47,13 +40,12 @@ class GsnapTest extends TestNGSuite with Matchers {
       }
     if (exitCode != 0)
       throw new SkipException("Skipping GSNAP test because the executable can not be found")
-    restoreConfig(oldMap)
   }
 
   @Test(description = "GSNAP version number capture from executable")
   def testVersion() = {
-    val oldMap = setConfig("db", "mock")
-    new Gsnap(null).getVersion should not be "N/A"
-    restoreConfig(oldMap)
+    new Gsnap(null) {
+      override def globalConfig = new Config(Map("db" -> "mock"))
+    }.getVersion should not be "N/A"
   }
 }

public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/config/Config.scala

@@ -21,11 +21,11 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
 /**
  * This class can store nested config values
- * @param map Map with value for new config
+ * @param _map Map with value for new config
  * @constructor Load config with existing map
  */
-class Config(var map: Map[String, Any],
-             protected[config] var defaults: Map[String, Any] = Map()) extends Logging {
+class Config(protected var _map: Map[String, Any],
+             protected var _defaults: Map[String, Any] = Map()) extends Logging {
   logger.debug("Init phase of config")
 
   /** Default constructor */
@@ -34,6 +34,9 @@ class Config(var map: Map[String, Any],
     loadDefaultConfig()
   }
 
+  def map = _map
+  def defaults = _defaults
+
   /**
    * Loading a environmental variable as location of config files to merge into the config
    * @param valueName Name of value
@@ -65,13 +68,13 @@ class Config(var map: Map[String, Any],
   def loadConfigFile(configFile: File, default: Boolean = false) {
     val configMap = fileToConfigMap(configFile)
     if (default) {
-      if (defaults.isEmpty) defaults = configMap
-      else defaults = mergeMaps(configMap, defaults)
-      logger.debug("New defaults: " + defaults)
+      if (_defaults.isEmpty) _defaults = configMap
+      else _defaults = mergeMaps(configMap, _defaults)
+      logger.debug("New defaults: " + _defaults)
     } else {
-      if (map.isEmpty) map = configMap
-      else map = mergeMaps(configMap, map)
-      logger.debug("New config: " + map)
+      if (_map.isEmpty) _map = configMap
+      else _map = mergeMaps(configMap, _map)
+      logger.debug("New config: " + _map)
     }
   }
 
@@ -84,11 +87,12 @@ class Config(var map: Map[String, Any],
    */
   def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
     val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
-    if (default) defaults = mergeMaps(valueMap, defaults)
-    else map = mergeMaps(valueMap, map)
+    if (default) _defaults = mergeMaps(valueMap, _defaults)
+    else _map = mergeMaps(valueMap, _map)
   }
 
   protected[config] var notFoundCache: List[ConfigValueIndex] = List()
+  protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
   protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
   protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
   protected[config] def clearCache(): Unit = {
@@ -103,24 +107,39 @@ class Config(var map: Map[String, Any],
    * @param s key
    * @return True if exist
    */
-  def contains(s: String): Boolean = map.contains(s)
+  def contains(s: String): Boolean = _map.contains(s)
+
+  /**
+   * Checks if value exist in config
+   * @param requestedIndex Index to value
+   * @return True if exist
+   */
+  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())
 
   /**
    * Checks if value exist in config
    * @param requestedIndex Index to value
+   * @param fixedValues Fixed values
    * @return True if exist
    */
-  def contains(requestedIndex: ConfigValueIndex): Boolean =
+  def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
     if (notFoundCache.contains(requestedIndex)) false
+    else if (fixedCache.contains(requestedIndex)) true
     else if (foundCache.contains(requestedIndex)) true
     else {
-      val value = Config.getValueFromMap(map, requestedIndex)
-      if (value.isDefined && value.get.value != None) {
-        foundCache += (requestedIndex -> value.get)
+      val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
+      if (fixedValue.isDefined) {
+        fixedCache += (requestedIndex -> fixedValue.get)
         true
       } else {
-        notFoundCache +:= requestedIndex
-        false
+        val value = Config.getValueFromMap(_map, requestedIndex)
+        if (value.isDefined && value.get.value != None) {
+          foundCache += (requestedIndex -> value.get)
+          true
+        } else {
+          notFoundCache +:= requestedIndex
+          false
+        }
       }
     }
 
@@ -132,9 +151,12 @@ class Config(var map: Map[String, Any],
    * @param freeVar Default true, if set false value must exist in module
    * @return True if exist
    */
-  def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
+  def contains(module: String, path: List[String],
+               key: String,
+               freeVar: Boolean = true,
+               fixedValues: Map[String, Any] = Map()): Boolean = {
     val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
-    contains(requestedIndex)
+    contains(requestedIndex, fixedValues)
   }
 
   /**
@@ -146,10 +168,23 @@ class Config(var map: Map[String, Any],
    * @param freeVar Default true, if set false value must exist in module
    * @return Config value
    */
-  protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
+  protected[config] def apply(module: String,
+                              path: List[String],
+                              key: String,
+                              default: Any = null,
+                              freeVar: Boolean = true,
+                              fixedValues: Map[String, Any] = Map()): ConfigValue = {
     val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
-    if (contains(requestedIndex)) foundCache(requestedIndex)
-    else if (default != null) {
+    if (contains(requestedIndex, fixedValues)) {
+      val fixedValue = fixedCache.get(requestedIndex)
+      if (fixedValue.isDefined) {
+        val userValue = Config.getValueFromMap(_map, requestedIndex)
+        if (userValue.isDefined)
+          logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
+      }
+
+      fixedValue.getOrElse(foundCache(requestedIndex))
+    } else if (default != null) {
       defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
       defaultCache(requestedIndex)
     } else ConfigValue(requestedIndex, null, null, freeVar)
@@ -179,9 +214,11 @@ class Config(var map: Map[String, Any],
 
     // Positions where values are found
     val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
+    val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
 
     // Positions where to start searching
     val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
+    val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
     val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
     val notFound = convertIndexValuesToMap(notFoundCache.map((_, None)), Some(false))
 
@@ -189,16 +226,19 @@ class Config(var map: Map[String, Any],
     val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
     val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)
 
-    writeMapToJsonFile(this.map, "input")
+    writeMapToJsonFile(_map, "input")
+    writeMapToJsonFile(_defaults, "defaults")
     writeMapToJsonFile(found, "found")
+    writeMapToJsonFile(fixed, "fixed")
     writeMapToJsonFile(effectiveFound, "effective.found")
+    writeMapToJsonFile(effectiveFixed, "effective.fixed")
     writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
     writeMapToJsonFile(notFound, "not.found")
     writeMapToJsonFile(fullEffective, "effective.full")
     writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
   }
 
-  override def toString: String = map.toString()
+  override def toString: String = _map.toString()
 }
 
 object Config extends Logging {
@@ -210,7 +250,7 @@ object Config extends Logging {
    * @param config2 Low prio map
    * @return Merged config
    */
-  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
+  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
 
   /**
    * Search for value in index position in a map

public/biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/config/Configurable.scala

@@ -15,6 +15,7 @@
  */
 package nl.lumc.sasc.biopet.utils.config
 
+import nl.lumc.sasc.biopet.utils.ConfigUtils
 import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
 
 trait Configurable extends ImplicitConversions {
@@ -35,9 +36,28 @@ trait Configurable extends ImplicitConversions {
   def configFullPath: List[String] = configPath ::: configName :: Nil
 
   /** Map to store defaults for config */
-  def defaults: Map[String, Any] = {
-    if (root != null) root.defaults
-    else globalConfig.defaults
+  def defaults: Map[String, Any] = Map()
+
+  /** This method merge defaults from the root to it's own */
+  protected def internalDefaults: Map[String, Any] = {
+    (root != null, defaults.isEmpty) match {
+      case (true, true)   => root.defaults
+      case (true, false)  => ConfigUtils.mergeMaps(defaults, root.defaults)
+      case (false, true)  => globalConfig.defaults
+      case (false, false) => ConfigUtils.mergeMaps(defaults, globalConfig.defaults)
+    }
+  }
+
+  /** All values found in this map will be skipped from the user config */
+  def fixedValues: Map[String, Any] = Map()
+
+  /** This method merge fixedValues from the root to it's own */
+  protected def internalFixedValues: Map[String, Any] = {
+    (root != null, fixedValues.isEmpty) match {
+      case (true, true)  => root.internalFixedValues
+      case (true, false) => ConfigUtils.mergeMaps(fixedValues, root.internalFixedValues)
+      case _             => fixedValues
+    }
   }
 
   val config = new ConfigFunctions
@@ -90,11 +110,11 @@ trait Configurable extends ImplicitConversions {
       val m = if (submodule != null) submodule else configName
       val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
       val d = {
-        val value = Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar))
+        val value = Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar))
         if (value.isDefined) value.get.value else default
       }
-      if (d == null) globalConfig(m, p, key, freeVar = freeVar)
-      else globalConfig(m, p, key, d, freeVar)
+      if (d == null) globalConfig(m, p, key, freeVar = freeVar, fixedValues = internalFixedValues)
+      else globalConfig(m, p, key, d, freeVar, fixedValues = internalFixedValues)
     }
 
     /**
@@ -117,7 +137,7 @@ trait Configurable extends ImplicitConversions {
       val m = if (submodule != null) submodule else configName
       val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
 
-      globalConfig.contains(m, p, key, freeVar) || Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
+      globalConfig.contains(m, p, key, freeVar, internalFixedValues) || Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
     }
   }
 }

public/biopet-utils/src/test/scala/nl/lumc/sasc/biopet/utils/config/ConfigurableTest.scala

@@ -25,10 +25,36 @@ import org.testng.annotations.Test
  * Created by pjvan_thof on 1/8/15.
  */
 class ConfigurableTest extends TestNGSuite with Matchers {
+
+  abstract class Cfg extends Configurable {
+    def get(key: String,
+            default: String = null,
+            submodule: String = null,
+            freeVar: Boolean = true,
+            sample: String = null,
+            library: String = null) = {
+      config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
+    }
+  }
+
+  class ClassA(val root: Configurable) extends Cfg
+
+  class ClassB(val root: Configurable) extends Cfg {
+    lazy val classA = new ClassA(this)
+    // Why this needs to be lazy?
+  }
+
+  class ClassC(val root: Configurable) extends Cfg {
+    def this() = this(null)
+    lazy val classB = new ClassB(this)
+    // Why this needs to be lazy?
+  }
+
   @Test def testConfigurable(): Unit = {
     val classC = new ClassC {
       override def configName = "classc"
       override val globalConfig = new Config(ConfigurableTest.map)
+      override val fixedValues = Map("fixed" -> "fixed")
     }
 
     classC.configPath shouldBe Nil
@@ -51,46 +77,33 @@ class ConfigurableTest extends TestNGSuite with Matchers {
     classC.get("bla", sample = "sample1", library = "library1").asString shouldBe "bla"
     classC.get("test", sample = "sample1", library = "library1").asString shouldBe "test"
     classC.get("test", sample = "sample1").asString shouldBe "test"
-  }
-}
 
-abstract class Cfg extends Configurable {
-  def get(key: String,
-          default: String = null,
-          submodule: String = null,
-          freeVar: Boolean = true,
-          sample: String = null,
-          library: String = null) = {
-    config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
+    // Fixed values
+    classC.get("fixed").asString shouldBe "fixed"
+    classC.classB.get("fixed").asString shouldBe "fixed"
+    classC.classB.classA.get("fixed").asString shouldBe "fixed"
   }
 }
 
-class ClassA(val root: Configurable) extends Cfg
-
-class ClassB(val root: Configurable) extends Cfg {
-  lazy val classA = new ClassA(this)
-  // Why this needs to be lazy?
-}
-
-class ClassC(val root: Configurable) extends Cfg {
-  def this() = this(null)
-  lazy val classB = new ClassB(this)
-  // Why this needs to be lazy?
-}
-
 object ConfigurableTest {
   val map = Map(
+    "fixed" -> "nonfixed",
     "classa" -> Map(
-      "k1" -> "a1"
+      "k1" -> "a1",
+      "fixed" -> "nonfixed"
     ), "classb" -> Map(
-      "k1" -> "b1"
+      "k1" -> "b1",
+      "fixed" -> "nonfixed"
     ), "classc" -> Map(
-      "k1" -> "c1"
+      "k1" -> "c1",
+      "fixed" -> "nonfixed"
     ), "samples" -> Map(
       "sample1" -> Map(
+        "fixed" -> "nonfixed",
         "test" -> "test",
         "libraries" -> Map(
           "library1" -> Map(
+            "fixed" -> "nonfixed",
             "bla" -> "bla"
           )
         )

public/carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala

@@ -38,12 +38,12 @@ class Carp(val root: Configurable) extends QScript with MultiSampleQScript with
   qscript =>
   def this() = this(null)
 
-  override def defaults = ConfigUtils.mergeMaps(Map(
+  override def defaults = Map(
     "mapping" -> Map(
       "skip_markduplicates" -> true,
       "aligner" -> "bwa-mem"
     )
-  ), super.defaults)
+  )
 
   def summaryFile = new File(outputDir, "Carp.summary.json")
 

public/gentrap/src/main/scala/nl/lumc/sasc/biopet/pipelines/gentrap/Gentrap.scala

@@ -100,24 +100,23 @@ class Gentrap(val root: Configurable) extends QScript
     })
 
   /** Default pipeline config */
-  override def defaults = ConfigUtils.mergeMaps(
-    Map(
-      "gsnap" -> Map(
-        "novelsplicing" -> 1,
-        "batch" -> 4,
-        "format" -> "sam"
-      ),
-      "cutadapt" -> Map("minimum_length" -> 20),
-      // avoid conflicts when merging since the MarkDuplicate tags often cause merges to fail
-      "picard" -> Map(
-        "programrecordid" -> "null"
-      ),
-      // disable markduplicates since it may not play well with all aligners (this can still be overriden via config)
-      "mapping" -> Map(
-        "skip_markduplicates" -> true,
-        "skip_metrics" -> true
-      )
-    ), super.defaults)
+  override def defaults = Map(
+    "gsnap" -> Map(
+      "novelsplicing" -> 1,
+      "batch" -> 4,
+      "format" -> "sam"
+    ),
+    "cutadapt" -> Map("minimum_length" -> 20),
+    // avoid conflicts when merging since the MarkDuplicate tags often cause merges to fail
+    "picard" -> Map(
+      "programrecordid" -> "null"
+    ),
+    // disable markduplicates since it may not play well with all aligners (this can still be overriden via config)
+    "mapping" -> Map(
+      "skip_markduplicates" -> true,
+      "skip_metrics" -> true
+    )
+  )
 
   /** Adds output merge jobs for the given expression mode */
   // TODO: can we combine the enum with the file extension (to reduce duplication and potential errors)

public/mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/Mapping.scala

@@ -97,13 +97,12 @@ class Mapping(val root: Configurable) extends QScript with SummaryQScript with S
   /** location of summary file */
   def summaryFile = new File(outputDir, sampleId.getOrElse("x") + "-" + libId.getOrElse("x") + ".summary.json")
 
-  override def defaults = ConfigUtils.mergeMaps(
-    Map(
-      "gsnap" -> Map(
-        "batch" -> 4,
-        "format" -> "sam"
-      )
-    ), super.defaults)
+  override def defaults = Map("gsnap" -> Map("batch" -> 4))
+
+  override def fixedValues = Map(
+    "gsnap" -> Map("format" -> "sam"),
+    "bowtie" -> Map("sam" -> true)
+  )
 
   /** File to add to the summary */
   def summaryFiles: Map[String, File] = Map("output_bamfile" -> finalBamFile, "input_R1" -> input_R1,

public/sage/src/main/scala/nl/lumc/sasc/biopet/pipelines/sage/Sage.scala

@@ -36,21 +36,22 @@ class Sage(val root: Configurable) extends QScript with MultiSampleQScript {
   var transcriptome: Option[File] = config("transcriptome")
   var tagsLibrary: Option[File] = config("tags_library")
 
-  override def defaults = ConfigUtils.mergeMaps(Map("bowtie" -> Map(
-    "m" -> 1,
-    "k" -> 1,
-    "best" -> true,
-    "strata" -> true,
-    "seedmms" -> 1
-  ), "mapping" -> Map(
-    "aligner" -> "bowtie",
-    "skip_flexiprep" -> true,
-    "skip_markduplicates" -> true
-  ), "flexiprep" -> Map(
-    "skip_clip" -> true,
-    "skip_trim" -> true
-  ), "strandSensitive" -> true
-  ), super.defaults)
+  override def defaults = Map(
+    "bowtie" -> Map(
+      "m" -> 1,
+      "k" -> 1,
+      "best" -> true,
+      "strata" -> true,
+      "seedmms" -> 1
+    ), "mapping" -> Map(
+      "aligner" -> "bowtie",
+      "skip_flexiprep" -> true,
+      "skip_markduplicates" -> true
+    ), "flexiprep" -> Map(
+      "skip_clip" -> true,
+      "skip_trim" -> true
+    ), "strandSensitive" -> true
+  )
 
   def summaryFile: File = new File(outputDir, "Sage.summary.json")
 

public/shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaVariantcallingTrait.scala

@@ -199,11 +199,11 @@ trait ShivaVariantcallingTrait extends SummaryQScript with SampleLibraryTag with
 
         val vcfFilter = new VcfFilter(qscript) {
           override def configName = "vcffilter"
-          override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
+          override def defaults = Map("min_sample_depth" -> 8,
             "min_alternate_depth" -> 2,
             "min_samples_pass" -> 1,
             "filter_ref_calls" -> true
-          ), super.defaults)
+          )
         }
         vcfFilter.inputVcf = m2v.output