Merge branch 'feature-config_unittest' into 'develop'

Feature config unittest

See also #89

Also some start for issue #55

See merge request !65

protected/biopet-gatk-pipelines/src/main/scala/nl/lumc/sasc/biopet/pipelines/gatk/GatkPipeline.scala

@@ -62,7 +62,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
   }
 
   val multisampleVariantcalling = new GatkVariantcalling(this) {
-    override protected lazy val configName = "gatkvariantcalling"
+    override def configName = "gatkvariantcalling"
     override def configPath: List[String] = "multisample" :: super.configPath
   }
 
@@ -97,7 +97,7 @@ class GatkPipeline(val root: Configurable) extends QScript with MultiSampleQScri
         val allRawVcfFiles = for ((sampleID, sampleOutput) <- samplesOutput) yield sampleOutput.variantcalling.rawFilterVcfFile
 
         val gatkVariantcalling = new GatkVariantcalling(this) {
-          override protected lazy val configName = "gatkvariantcalling"
+          override def configName = "gatkvariantcalling"
           override def configPath: List[String] = "multisample" :: super.configPath
         }
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/MultiSampleQScript.scala

@@ -15,7 +15,7 @@
  */
 package nl.lumc.sasc.biopet.core
 
-import nl.lumc.sasc.biopet.core.config.{ Config, Configurable }
+import nl.lumc.sasc.biopet.core.config.{ ConfigValue, Config, Configurable }
 import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
 trait MultiSampleQScript extends BiopetQScript {
@@ -29,31 +29,61 @@ trait MultiSampleQScript extends BiopetQScript {
     def getLibrary(key: String) = libraries(key)
   }
 
-  var samplesConfig: Map[String, Any] = config("samples")
-  var samplesOutput: Map[String, SampleOutput] = Map()
+  if (!config.contains("samples")) logger.warn("No Samples found in config")
+
+  /**
+   * Returns a map with all sample configs
+   */
+  val getSamplesConfig: Map[String, Any] = config("samples", default = Map())
+
+  /**
+   * Returns a list of all sampleIDs
+   */
+  val getSamples: Set[String] = getSamplesConfig.keySet
+
+  /**
+   * Returns the global sample directory
+   * @return global sample directory
+   */
   def globalSampleDir: String = outputDir + "samples/"
 
+  var samplesOutput: Map[String, SampleOutput] = Map()
+
+  /**
+   * Runs runSingleSampleJobs method for each sample
+   */
   final def runSamplesJobs() {
-    if (samplesConfig == null) samplesConfig = Map()
-    if (Config.global.contains("samples")) for ((key, value) <- samplesConfig) {
+    for ((key, value) <- getSamplesConfig) {
       var sample = any2map(value)
       if (!sample.contains("ID")) sample += ("ID" -> key)
       if (sample("ID") == key) {
+        currentSample = key
         samplesOutput += key -> runSingleSampleJobs(sample)
+        currentSample = null
       } else logger.warn("Key is not the same as ID on value for sample")
     }
-    else logger.warn("No Samples found in config")
   }
 
   def runSingleSampleJobs(sampleConfig: Map[String, Any]): SampleOutput
+
+  /**
+   * Run sample with only sampleID
+   * @param sample sampleID
+   * @return
+   */
   def runSingleSampleJobs(sample: String): SampleOutput = {
-    var map = any2map(samplesConfig(sample))
+    var map = any2map(getSamplesConfig(sample))
     if (map.contains("ID") && map("ID") != sample)
       throw new IllegalStateException("ID in config not the same as the key")
     else map += ("ID" -> sample)
     return runSingleSampleJobs(map)
   }
 
+  /**
+   * Runs runSingleLibraryJobs method for each library found in sampleConfig
+   * @param sampleConfig sample config
+   * @return Map with libraryID -> LibraryOutput object
+   */
   final def runLibraryJobs(sampleConfig: Map[String, Any]): Map[String, LibraryOutput] = {
     var output: Map[String, LibraryOutput] = Map()
     val sampleID = sampleConfig("ID").toString
@@ -63,11 +93,88 @@ trait MultiSampleQScript extends BiopetQScript {
         var library = any2map(value)
         if (!library.contains("ID")) library += ("ID" -> key)
         if (library("ID") == key) {
+          currentLibrary = key
           output += key -> runSingleLibraryJobs(library, sampleConfig)
+          currentLibrary = null
         } else logger.warn("Key is not the same as ID on value for run of sample: " + sampleID)
       }
     } else logger.warn("No runs found in config for sample: " + sampleID)
     return output
   }
   def runSingleLibraryJobs(runConfig: Map[String, Any], sampleConfig: Map[String, Any]): LibraryOutput
+
+  protected var currentSample: String = null
+  protected var currentLibrary: String = null
+
+  /**
+   * Set current sample manual, only use this when not using runSamplesJobs method
+   * @param sample
+   */
+  def setCurrentSample(sample: String) {
+    logger.debug("Manual sample set to: " + sample)
+    currentSample = sample
+  }
+
+  /**
+   * Gets current sample
+   * @return current sample
+   */
+  def getCurrentSample = currentSample
+
+  /**
+   * Reset current sample manual, only use this when not using runSamplesJobs method
+   */
+  def resetCurrentSample() {
+    logger.debug("Manual sample reset")
+    currentSample = null
+  }
+
+  /**
+   * Set current library manual, only use this when not using runLibraryJobs method
+   * @param library
+   */
+  def setCurrentLibrary(library: String) {
+    logger.debug("Manual library set to: " + library)
+    currentLibrary = library
+  }
+
+  /**
+   * Gets current library
+   * @return current library
+   */
+  def getCurrentLibrary = currentLibrary
+
+  /**
+   * Reset current library manual, only use this when not using runLibraryJobs method
+   */
+  def resetCurrentLibrary() {
+    logger.debug("Manual library reset")
+    currentLibrary = null
+  }
+
+  override protected[core] def configFullPath: List[String] = {
+    (if (currentSample != null) "samples" :: currentSample :: Nil else Nil) :::
+      (if (currentLibrary != null) "libraries" :: currentLibrary :: Nil else Nil) :::
+      super.configFullPath
+  }
+
+  protected class ConfigFunctions extends super.ConfigFunctions {
+    override def apply(key: String,
+                       default: Any = null,
+                       submodule: String = null,
+                       required: Boolean = false,
+                       freeVar: Boolean = true,
+                       sample: String = currentSample,
+                       library: String = currentLibrary): ConfigValue = {
+      super.apply(key, default, submodule, required, freeVar, sample, library)
+    }
+
+    override def contains(key: String,
+                          submodule: String = null,
+                          freeVar: Boolean = true,
+                          sample: String = currentSample,
+                          library: String = currentLibrary) = {
+      super.contains(key, submodule, freeVar, sample, library)
+    }
+  }
 }

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/ToolCommand.scala

@@ -16,10 +16,10 @@
 package nl.lumc.sasc.biopet.core
 
 trait ToolCommand extends MainCommand with Logging {
-  abstract class AbstractArgs {
+  protected abstract class AbstractArgs {
   }
 
-  abstract class AbstractOptParser extends scopt.OptionParser[Args](commandName) {
+  protected abstract class AbstractOptParser extends scopt.OptionParser[Args](commandName) {
     opt[String]('l', "log_level") foreach { x =>
       x.toLowerCase match {
         case "debug" => logger.setLevel(org.apache.log4j.Level.DEBUG)
@@ -44,6 +44,6 @@ trait ToolCommand extends MainCommand with Logging {
     } text ("Print version")
   }
 
-  type Args <: AbstractArgs
-  type OptParser <: AbstractOptParser
+  protected type Args <: AbstractArgs
+  protected type OptParser <: AbstractOptParser
 }
\ No newline at end of file

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/Config.scala

@@ -19,18 +19,31 @@ import java.io.File
 import nl.lumc.sasc.biopet.core.Logging
 import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
+/**
+ * This class can store nested config values
+ * @param map Map with value for new config
+ * @constructor Load config with existing map
+ */
 class Config(var map: Map[String, Any]) extends Logging {
   logger.debug("Init phase of config")
+
+  /**
+   * Default constructor
+   */
   def this() = {
     this(Map())
     loadDefaultConfig()
   }
 
+  /**
+   * Loading a environmental variable as location of config files to merge into the config
+   * @param valueName Name of value
+   */
   def loadConfigEnv(valueName: String) {
     val globalFiles = sys.env.get(valueName).getOrElse("").split(":")
     if (globalFiles.isEmpty) logger.info(valueName + " value not found, no global config is loaded")
     for (globalFile <- globalFiles) {
-      var file: File = new File(globalFile)
+      val file: File = new File(globalFile)
       if (file.exists()) {
         logger.info("Loading config file: " + file)
         loadConfigFile(file)
@@ -38,10 +51,17 @@ class Config(var map: Map[String, Any]) extends Logging {
     }
   }
 
+  /**
+   * Loading default value for biopet
+   */
   def loadDefaultConfig() {
     loadConfigEnv("BIOPET_CONFIG")
   }
 
+  /**
+   * Merge a json file into the config
+   * @param configFile Location of file
+   */
   def loadConfigFile(configFile: File) {
     val configMap = fileToConfigMap(configFile)
 
@@ -53,12 +73,26 @@ class Config(var map: Map[String, Any]) extends Logging {
   protected[config] var notFoundCache: List[ConfigValueIndex] = List()
   protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
   protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
+  protected[config] def clearCache: Unit = {
+    notFoundCache = List()
+    foundCache = Map()
+    defaultCache = Map()
+  }
 
+  /**
+   * Check if value exist in root of config
+   * @deprecated
+   * @param s key
+   * @return True if exist
+   */
   def contains(s: String): Boolean = map.contains(s)
-  def contains(requestedIndex: ConfigValueIndex, freeVar: Boolean): Boolean = contains(requestedIndex.module, requestedIndex.path, requestedIndex.key, freeVar)
-  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex.module, requestedIndex.path, requestedIndex.key, true)
-  def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
-    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
+
+  /**
+   * Checks if value exist in config
+   * @param requestedIndex Index to value
+   * @return True if exist
+   */
+  def contains(requestedIndex: ConfigValueIndex): Boolean =
     if (notFoundCache.contains(requestedIndex)) return false
     else if (foundCache.contains(requestedIndex)) return true
     else {
@@ -71,22 +105,45 @@ class Config(var map: Map[String, Any]) extends Logging {
         return false
       }
     }
+
+  /**
+   * Checks if value exist in config
+   * @param module Name of module
+   * @param path Path to start searching
+   * @param key Name of value
+   * @param freeVar Default true, if set false value must exist in module
+   * @return True if exist
+   */
+  def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
+    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
+    contains(requestedIndex)
   }
 
+  /**
+   * Find value in config
+   * @param module Name of module
+   * @param path Path to start searching
+   * @param key Name of value
+   * @param default Default value when no value is found
+   * @param freeVar Default true, if set false value must exist in module
+   * @return Config value
+   */
   protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
-    val requestedIndex = ConfigValueIndex(module, path, key)
-    if (contains(requestedIndex, freeVar)) return foundCache(requestedIndex)
+    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
+    if (contains(requestedIndex)) return foundCache(requestedIndex)
     else if (default != null) {
-      defaultCache += (requestedIndex -> ConfigValue.apply(requestedIndex, null, default, true))
+      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
       return defaultCache(requestedIndex)
-    } else {
-      logger.error("Value in config could not be found but it seems required, index: " + requestedIndex)
-      throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
-    }
+    } else throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
   }
 
+  //TODO: New version of report is needed
+  /**
+   * Makes report for all used values
+   * @return Config report
+   */
   def getReport: String = {
-    var output: StringBuilder = new StringBuilder
+    val output: StringBuilder = new StringBuilder
     output.append("Config report, sorted on module:\n")
     var modules: Map[String, StringBuilder] = Map()
     for ((key, value) <- foundCache) {
@@ -118,43 +175,60 @@ class Config(var map: Map[String, Any]) extends Logging {
 object Config extends Logging {
   val global = new Config
 
+  /**
+   * Merge 2 config objects
+   * @param config1 prio over config 2
+   * @param config2
+   * @return Merged config
+   */
   def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
 
-  private def getMapFromPath(map: Map[String, Any], path: List[String]): Map[String, Any] = {
-    var returnMap: Map[String, Any] = map
-    for (m <- path) {
-      if (!returnMap.contains(m)) return Map()
-      else returnMap = any2map(returnMap(m))
+  /**
+   * Search for value in index position in a map
+   * @param map Map to search in
+   * @param startIndex Config index
+   * @return Value
+   */
+  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
+    def getFromPath(path: List[String]): Option[ConfigValue] = {
+      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
+      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
+      else if (startIndex.freeVar) {
+        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
+        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
+        else None
+      } else None
     }
-    return returnMap
-  }
 
-  def getValueFromMap(map: Map[String, Any], index: ConfigValueIndex): Option[ConfigValue] = {
-    var submodules = index.path.reverse
-    while (!submodules.isEmpty) {
-      var submodules2 = submodules
-      while (!submodules2.isEmpty) {
-        val p = getMapFromPath(map, submodules2 ::: index.module :: Nil)
-        if (p.contains(index.key)) {
-          return Option(ConfigValue(index, ConfigValueIndex(index.module, submodules2, index.key), p(index.key)))
-        }
-        if (index.freeVar) {
-          val p2 = getMapFromPath(map, submodules2)
-          if (p2.contains(index.key)) {
-            return Option(ConfigValue(index, ConfigValueIndex(index.module, submodules2, index.key), p2(index.key)))
-          }
-        }
-        submodules2 = submodules2.init
+    def tailSearch(path: List[String]): Option[ConfigValue] = {
+      val p = getFromPath(path)
+      if (p != None) p
+      else if (path == Nil) None
+      else {
+        val p = initSearch(path)
+        if (p.isDefined) p
+        else tailSearch(path.tail)
       }
-      submodules = submodules.tail
     }
-    val p = getMapFromPath(map, index.module :: Nil)
-    if (p.contains(index.key)) { // Module is not nested
-      return Option(ConfigValue(index, ConfigValueIndex(index.module, Nil, index.key), p(index.key)))
-    } else if (map.contains(index.key) && index.freeVar) { // Root value of json
-      return Option(ConfigValue(index, ConfigValueIndex("", Nil, index.key), map(index.key)))
-    } else { // At this point key is not found on the path
-      return None
+
+    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
+      val p = getFromPath(path)
+      if (p.isDefined) p
+      else if (path == Nil) None
+      else {
+        val p = skipNested(path, tail)
+        if (p.isDefined) p
+        else initSearch(path.init, path.last :: tail)
+      }
+    }
+
+    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
+      val p = getFromPath(path ::: tail)
+      if (p.isDefined) p
+      else if (tail == Nil) None
+      else skipNested(path, tail.tail)
     }
+
+    return tailSearch(startIndex.path)
   }
 }
\ No newline at end of file

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/ConfigValue.scala

@@ -19,15 +19,58 @@ import java.io.File
 import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
 class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValueIndex, val value: Any, val default: Boolean) {
+  /**
+   * Get value as String
+   * @return value as String
+   */
   def asString = any2string(value)
+
+  /**
+   * Get value as Int
+   * @return value as Int
+   */
   def asInt = any2int(value)
+
+  /**
+   * Get value as Double
+   * @return value as Double
+   */
   def asDouble = any2double(value)
+
+  /**
+   * Get value as List[Any]
+   * @return value as List[Any]
+   */
   def asList = any2list(value)
+
+  /**
+   * Get value as List[File]
+   * @return value as List[File]
+   */
   def asFileList: List[File] = for (file <- any2stringList(value)) yield new File(file)
+
+  /**
+   * Get value as List[String]
+   * @return value as List[String]
+   */
   def asStringList: List[String] = any2stringList(value)
+
+  /**
+   * Get value as Map
+   * @return value as Map
+   */
   def asMap = any2map(value)
+
+  /**
+   * Get value as Boolean
+   * @return value as Boolean
+   */
   def asBoolean = any2boolean(value)
 
+  /**
+   * Readable output of indexes and value, just for debug
+   * @return
+   */
   override def toString: String = {
     var output = "key = " + requestIndex.key
     output += ", value = " + value
@@ -41,9 +84,25 @@ class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValu
 }
 
 object ConfigValue {
+  /**
+   *
+   * @param requestIndex Index where to start searching
+   * @param foundIndex Index where value is found
+   * @param value Found value
+   * @return ConfigValue object
+   */
   def apply(requestIndex: ConfigValueIndex, foundIndex: ConfigValueIndex, value: Any) = {
     new ConfigValue(requestIndex, foundIndex, value, false)
   }
+
+  /**
+   *
+   * @param requestIndex Index where to start searching
+   * @param foundIndex Index where value is found
+   * @param value Found value
+   * @param default Value is a default value
+   * @return ConfigValue object
+   */
   def apply(requestIndex: ConfigValueIndex, foundIndex: ConfigValueIndex, value: Any, default: Boolean) = {
     new ConfigValue(requestIndex, foundIndex, value, default)
   }

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/ConfigValueIndex.scala

@@ -15,15 +15,13 @@
  */
 package nl.lumc.sasc.biopet.core.config
 
-class ConfigValueIndex(val module: String, val path: List[String], val key: String, val freeVar: Boolean = true) {
+/**
+ * General case class used as index config values. This stores the path to the value, the module, name of the value and if freeVar is allowed
+ * @param module Module where this value is belonging to
+ * @param path Path to value
+ * @param key Name of value
+ * @param freeVar Default true, if set false value must exist in module
+ */
+case class ConfigValueIndex(module: String, path: List[String], key: String, freeVar: Boolean = true) {
   override def toString = "Module = " + module + ",  path = " + path + ",  key = " + key + ",  freeVar = " + freeVar
-}
-
-object ConfigValueIndex {
-  private var cache: Map[(String, List[String], String), ConfigValueIndex] = Map()
-
-  def apply(module: String, path: List[String], key: String, freeVar: Boolean = true): ConfigValueIndex = {
-    if (!cache.contains(module, path, key)) cache += ((module, path, key) -> new ConfigValueIndex(module, path, key, freeVar))
-    return cache(module, path, key)
-  }
 }
\ No newline at end of file

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/Configurable.scala

@@ -20,36 +20,105 @@ import nl.lumc.sasc.biopet.core.Logging
 import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
 
 trait Configurable extends ImplicitConversions {
+  /**
+   * Should be object of parant object
+   */
   val root: Configurable
+
+  /**
+   * Get default path to search config values for current object
+   * @return
+   */
   def configPath: List[String] = if (root != null) root.configFullPath else List()
-  protected lazy val configName = getClass.getSimpleName.toLowerCase
-  protected lazy val configFullPath = configName :: configPath
+
+  /**
+   * Gets name of module for config
+   * @return
+   */
+  protected[core] def configName = getClass.getSimpleName.toLowerCase