Merge branch 'feature-yaml_support' into 'develop'

Feature yaml support

Issue #142

Now implemented a yaml parser and commandline argument to add values to config. Json still uses the old parser, when a file extension is ".yaml" the new parser is used. Later we might remove the old parser completely since SnakeYaml also supports json.

See merge request !146

public/biopet-framework/pom.xml

@@ -110,5 +110,10 @@
             <artifactId>scopt_2.10</artifactId>
             <version>3.3.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.15</version>
+        </dependency>
     </dependencies>
 </project>

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -25,19 +25,19 @@ import org.broadinstitute.gatk.queue.function.scattergather.ScatterGatherableFun
 import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
 import scala.collection.mutable.ListBuffer
 
-/**
- * Base for biopet pipeline
- */
+/** Base for biopet pipeline */
 trait BiopetQScript extends Configurable with GatkLogging {
 
-  @Argument(doc = "JSON config file(s)", fullName = "config_file", shortName = "config", required = false)
+  @Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
   val configfiles: List[File] = Nil
 
+  @Argument(doc = "Config values, value should be formatted like 'key=value' or 'path:path:key=value'", fullName = "config_value", shortName = "cv", required = false)
+  val configValues: List[String] = Nil
+
+  /** Output directory of pipeline */
   var outputDir: File = {
-    Config.getValueFromMap(globalConfig.map, ConfigValueIndex(this.configName, configPath, "output_dir")) match {
-      case Some(value) => new File(value.asString).getAbsoluteFile
-      case _           => new File(".")
-    }
+    if (config.contains("output_dir", path = Nil)) config("output_dir", path = Nil).asFile
+    else new File(".")
   }
 
   @Argument(doc = "Disable all scatters", shortName = "DSC", required = false)
@@ -57,11 +57,10 @@ trait BiopetQScript extends Configurable with GatkLogging {
   /** Pipeline itself */
   def biopetScript
 
-  /**
-   * Script from queue itself, final to force some checks for each pipeline and write report
-   */
+  /** Script from queue itself, final to force some checks for each pipeline and write report */
   final def script() {
-    outputDir = config("output_dir").asFile.getAbsoluteFile
+    outputDir = config("output_dir")
+    outputDir = outputDir.getAbsoluteFile
     init
     biopetScript
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/PipelineCommand.scala

@@ -20,9 +20,7 @@ import java.io.File
 import nl.lumc.sasc.biopet.core.config.Config
 import nl.lumc.sasc.biopet.core.workaround.BiopetQCommandLine
 
-/**
- * Wrapper around executable from Queue
- */
+/** Wrapper around executable from Queue */
 trait PipelineCommand extends MainCommand with GatkLogging {
 
   /**
@@ -31,10 +29,7 @@ trait PipelineCommand extends MainCommand with GatkLogging {
    */
   def pipeline = "/" + getClass.getName.stripSuffix("$").replaceAll("\\.", "/") + ".class"
 
-  /**
-   * Class can be used directly from java with -cp option
-   * @param args
-   */
+  /** Class can be used directly from java with -cp option */
   def main(args: Array[String]): Unit = {
     val argsSize = args.size
     for (t <- 0 until argsSize) {
@@ -42,6 +37,17 @@ trait PipelineCommand extends MainCommand with GatkLogging {
         if (t >= argsSize) throw new IllegalStateException("-config needs a value")
         Config.global.loadConfigFile(new File(args(t + 1)))
       }
+
+      if (args(t) == "-cv" || args(t) == "--config_value") {
+        val v = args(t + 1).split("=")
+        require(v.size == 2, "Value should be formatted like 'key=value' or 'path:path:key=value'")
+        val value = v(1)
+        val p = v(0).split(":")
+        val key = p.last
+        val path = p.dropRight(1).toList
+        Config.global.addValue(key, value, path)
+      }
+
       if (args(t) == "--logging_level" || args(t) == "-l") {
         args(t + 1).toLowerCase match {
           case "debug" => Logging.logger.setLevel(org.apache.log4j.Level.DEBUG)

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/Config.scala

@@ -20,19 +20,16 @@ import nl.lumc.sasc.biopet.core.Logging
 import nl.lumc.sasc.biopet.utils.ConfigUtils
 import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
-import scala.reflect.io.Directory
-
 /**
  * This class can store nested config values
  * @param map Map with value for new config
  * @constructor Load config with existing map
  */
-class Config(var map: Map[String, Any]) extends Logging {
+class Config(var map: Map[String, Any],
+             protected[core] var defaults: Map[String, Any] = Map()) extends Logging {
   logger.debug("Init phase of config")
 
-  /**
-   * Default constructor
-   */
+  /** Default constructor */
   def this() = {
     this(Map())
     loadDefaultConfig()
@@ -41,15 +38,16 @@ class Config(var map: Map[String, Any]) extends Logging {
   /**
    * Loading a environmental variable as location of config files to merge into the config
    * @param valueName Name of value
+   * @param default if true files are added to default instead of normal map
    */
-  def loadConfigEnv(valueName: String) {
+  def loadConfigEnv(valueName: String, default: Boolean) {
     sys.env.get(valueName) match {
       case Some(globalFiles) => {
         for (globalFile <- globalFiles.split(":")) {
           val file: File = new File(globalFile)
           if (file.exists) {
             logger.info("Loading config file: " + file)
-            loadConfigFile(file)
+            loadConfigFile(file, default)
           } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
         }
       }
@@ -57,23 +55,39 @@ class Config(var map: Map[String, Any]) extends Logging {
     }
   }
 
-  /**
-   * Loading default value for biopet
-   */
+  /** Loading default value for biopet */
   def loadDefaultConfig() {
-    loadConfigEnv("BIOPET_CONFIG")
+    loadConfigEnv("BIOPET_CONFIG", true)
   }
 
   /**
    * Merge a json file into the config
    * @param configFile Location of file
    */
-  def loadConfigFile(configFile: File) {
+  def loadConfigFile(configFile: File, default: Boolean = false) {
     val configMap = fileToConfigMap(configFile)
+    if (default) {
+      if (defaults.isEmpty) defaults = configMap
+      else defaults = mergeMaps(configMap, defaults)
+      logger.debug("New defaults: " + defaults)
+    } else {
+      if (map.isEmpty) map = configMap
+      else map = mergeMaps(configMap, map)
+      logger.debug("New config: " + map)
+    }
+  }
 
-    if (map.isEmpty) map = configMap
-    else map = mergeMaps(configMap, map)
-    logger.debug("New config: " + map)
+  /**
+   * Add a single vallue to the config
+   * @param key key of value
+   * @param value value itself
+   * @param path Path to value
+   * @param default if true value is put in default map
+   */
+  def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
+    val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
+    if (default) defaults = mergeMaps(valueMap, defaults)
+    else map = mergeMaps(valueMap, map)
   }
 
   protected[config] var notFoundCache: List[ConfigValueIndex] = List()

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/ConfigValue.scala

@@ -19,64 +19,34 @@ import java.io.File
 import nl.lumc.sasc.biopet.utils.ConfigUtils._
 
 class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValueIndex, val value: Any, val default: Boolean) {
-  /**
-   * Get value as String
-   * @return value as String
-   */
+  /** Get value as String */
   def asString = any2string(value)
 
-  /**
-   * Get value as File
-   * @return value as File
-   */
+  /** Get value as File */
   def asFile = new File(any2string(value))
 
-  /**
-   * Get value as Int
-   * @return value as Int
-   */
+  /** Get value as Int */
   def asInt = any2int(value)
 
-  /**
-   * Get value as Double
-   * @return value as Double
-   */
+  /** Get value as Double */
   def asDouble = any2double(value)
 
-  /**
-   * Get value as List[Any]
-   * @return value as List[Any]
-   */
+  /** Get value as List[Any] */
   def asList = any2list(value)
 
-  /**
-   * Get value as List[File]
-   * @return value as List[File]
-   */
+  /** Get value as List[File] */
   def asFileList: List[File] = for (file <- any2stringList(value)) yield new File(file)
 
-  /**
-   * Get value as List[String]
-   * @return value as List[String]
-   */
+  /** Get value as List[String] */
   def asStringList: List[String] = any2stringList(value)
 
-  /**
-   * Get value as Map
-   * @return value as Map
-   */
+  /** Get value as Map */
   def asMap = any2map(value)
 
-  /**
-   * Get value as Boolean
-   * @return value as Boolean
-   */
+  /** Get value as Boolean */
   def asBoolean = any2boolean(value)
 
-  /**
-   * Readable output of indexes and value, just for debug
-   * @return
-   */
+  /** Readable output of indexes and value, just for debug */
   override def toString: String = {
     var output = "key = " + requestIndex.key
     output += ", value = " + value

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/Configurable.scala

@@ -15,9 +15,7 @@
  */
 package nl.lumc.sasc.biopet.core.config
 
-import nl.lumc.sasc.biopet.core.Logging
 import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
-import scala.collection.JavaConversions._
 
 trait Configurable extends ImplicitConversions {
   /** Should be object of parant object */
@@ -39,7 +37,7 @@ trait Configurable extends ImplicitConversions {
   /** Map to store defaults for config */
   def defaults: Map[String, Any] = {
     if (root != null) root.defaults
-    else Map()
+    else globalConfig.defaults
   }
 
   val config = new ConfigFunctions
@@ -52,15 +50,13 @@ trait Configurable extends ImplicitConversions {
    * @param submodule
    * @return
    */
-  def path(sample: String = null, library: String = null, submodule: String = null) = {
+  def getConfigPath(sample: String = null, library: String = null, submodule: String = null) = {
     (if (sample != null) "samples" :: sample :: Nil else Nil) :::
       (if (library != null) "libraries" :: library :: Nil else Nil) :::
       (if (submodule != null) configPath ::: configName :: Nil else configPath)
   }
 
-  /**
-   * Class is used for retrieval of config values
-   */
+  /** Class is used for retrieval of config values */
   protected class ConfigFunctions(val defaultSample: Option[String] = None, val defaultLibrary: Option[String] = None) {
     def this(defaultSample: String, defaultLibrary: String) = {
       this(defaultSample = Some(defaultSample), defaultLibrary = Some(defaultLibrary))
@@ -91,11 +87,12 @@ trait Configurable extends ImplicitConversions {
               submodule: String = null,
               freeVar: Boolean = true,
               sample: String = null,
-              library: String = null): ConfigValue = {
+              library: String = null,
+              path: List[String] = null): ConfigValue = {
       val s = if (sample != null || defaultSample.isEmpty) sample else defaultSample.get
       val l = if (library != null || defaultLibrary.isEmpty) library else defaultLibrary.get
       val m = if (submodule != null) submodule else configName
-      val p = path(s, l, submodule)
+      val p = if (path == null) getConfigPath(s, l, submodule) else path
       val d = {
         val value = Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar))
         if (value.isDefined) value.get.value else default
@@ -117,11 +114,12 @@ trait Configurable extends ImplicitConversions {
                  submodule: String = null,
                  freeVar: Boolean = true,
                  sample: String = null,
-                 library: String = null) = {
+                 library: String = null,
+                 path: List[String] = null) = {
       val s = if (sample != null || defaultSample.isEmpty) sample else defaultSample.get
       val l = if (library != null || defaultLibrary.isEmpty) library else defaultLibrary.get
       val m = if (submodule != null) submodule else configName
-      val p = path(s, l, submodule)
+      val p = if (path == null) getConfigPath(s, l, submodule) else path
 
       globalConfig.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
     }

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/utils/ConfigUtils.scala

@@ -20,7 +20,9 @@ import nl.lumc.sasc.biopet.core.BiopetQScript
 import nl.lumc.sasc.biopet.core.Logging
 import nl.lumc.sasc.biopet.core.config.ConfigValue
 import argonaut._, Argonaut._
+import org.yaml.snakeyaml.Yaml
 import scalaz._, Scalaz._
+import scala.collection.JavaConversions._
 
 /**
  * This object contains general function for the config
@@ -78,16 +80,13 @@ object ConfigUtils extends Logging {
     val value = map.get(path.head)
     if (path.tail == Nil || value == None) value
     else value.get match {
-      case map: Map[_, _] => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
-      case _              => None
+      case map: Map[_, _]                     => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
+      case map: java.util.LinkedHashMap[_, _] => getValueFromPath(map.toMap.asInstanceOf[Map[String, Any]], path.tail)
+      case _                                  => None
     }
   }
 
-  /**
-   * Make json aboject from a file
-   * @param configFile Input file
-   * @return Json object
-   */
+  /** Make json aboject from a file */
   def fileToJson(configFile: File): Json = {
     logger.debug("Jsonfile: " + configFile)
     val jsonText = scala.io.Source.fromFile(configFile).mkString
@@ -99,22 +98,25 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert config value to map
-   * @param configFile
-   * @return Config map
-   */
+  /** Convert config value to map */
   def fileToConfigMap(configFile: File): Map[String, Any] = {
-    val configJson = jsonToMap(fileToJson(configFile))
-    logger.debug("Contain: " + configJson)
-    return configJson
+
+    val configMap = {
+      if (configFile.getName.endsWith(".yaml")) yamlToMap(configFile)
+      else jsonToMap(fileToJson(configFile))
+    }
+    logger.debug("Contain: " + configMap)
+    return configMap
   }
 
-  /**
-   * Convert json to native scala map/values
-   * @param json input json
-   * @return
-   */
+  /** Convert a yaml file to map[String, Any] */
+  def yamlToMap(file: File): Map[String, Any] = {
+    val yaml = new Yaml()
+    val a = yaml.load(scala.io.Source.fromFile(file).reader())
+    ConfigUtils.any2map(a)
+  }
+
+  /** Convert json to native scala map/values */
   def jsonToMap(json: Json): Map[String, Any] = {
     var output: Map[String, Any] = Map()
     if (json.isObject) {
@@ -126,11 +128,7 @@ object ConfigUtils extends Logging {
     return output
   }
 
-  /**
-   * Convert json value to native scala value
-   * @param json input json
-   * @return
-   */
+  /** Convert json value to native scala value */
   def jsonToAny(json: Json): Any = {
     if (json.isObject) return jsonToMap(json)
     else if (json.isArray) {
@@ -147,11 +145,7 @@ object ConfigUtils extends Logging {
     else throw new IllegalStateException("Config value type not supported, value: " + json)
   }
 
-  /**
-   * Convert native scala map to json
-   * @param map Input map
-   * @return
-   */
+  /** Convert native scala map to json */
   def mapToJson(map: Map[String, Any]): Json = {
     map.foldLeft(jEmptyObject)((acc, kv) => (kv._1 := {
       kv._2 match {
@@ -161,11 +155,7 @@ object ConfigUtils extends Logging {
     }) ->: acc)
   }
 
-  /**
-   * Convert native scala value to json, fall back on .toString if type is not a native scala value
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert native scala value to json, fall back on .toString if type is not a native scala value */
   def anyToJson(any: Any): Json = {
     any match {
       case j: Json      => j
@@ -183,11 +173,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to String
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to String */
   def any2string(any: Any): String = {
     if (any == null) return null
     any match {
@@ -196,11 +182,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to Int
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Int */
   def any2int(any: Any): Int = {
     any match {
       case i: Int    => i
@@ -214,11 +196,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to Long
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Long */
   def any2long(any: Any): Long = {
     any match {
       case l: Double => l.toLong
@@ -232,11 +210,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to Double
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Double */
   def any2double(any: Any): Double = {
     any match {
       case d: Double => d
@@ -251,11 +225,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to Float
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Float */
   def any2float(any: Any): Float = {
     any match {
       case f: Double => f.toFloat
@@ -270,11 +240,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to Boolean
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Boolean */
   def any2boolean(any: Any): Boolean = {
     any match {
       case b: Boolean => b
@@ -290,11 +256,7 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to List[Any], fallback on list with 1 value
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to List[Any], fallback on list with 1 value */
   def any2list(any: Any): List[Any] = {
     if (any == null) return null
     any match {
@@ -303,42 +265,39 @@ object ConfigUtils extends Logging {
     }
   }
 
-  /**
-   * Convert Any to List[String]
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to List[String] */
   def any2stringList(any: Any): List[String] = {
     if (any == null) return null
     any2list(any).map(_.toString)
   }
 
-  /**
-   * Convert Any to List[File]
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to List[File] */
   def any2fileList(any: Any): List[File] = {
     if (any == null) return null
     any2list(any).map(x => new File(x.toString))
   }
 
-  /**
-   * Convert Any to Map[String, Any]
-   * @param any Input Any value
-   * @return
-   */
+  /** Convert Any to Map[String, Any] */
   def any2map(any: Any): Map[String, Any] = {
     if (any == null) return null
     any match {
-      case m: Map[_, _] => m.map(x => x._1.toString -> x._2)
-      case _            => throw new IllegalStateException("Value '" + any + "' is not an Map")
+      case m: Map[_, _]                     => m.map(x => x._1.toString -> x._2)
+      case m: java.util.LinkedHashMap[_, _] => nestedJavaHashMaptoScalaMap(m)
+      case _                                => throw new IllegalStateException("Value '" + any + "' is not an Map")
     }
   }
 
-  /**
-   * Trait for implicit conversions for ConfigValue to native scala values
-   */
+  /** Convert nested java hash map to scala hash map */
+  def nestedJavaHashMaptoScalaMap(input: java.util.LinkedHashMap[_, _]): Map[String, Any] = {
+    input.map(value => {
+      value._2 match {
+        case m: java.util.LinkedHashMap[_, _] => value._1.toString -> nestedJavaHashMaptoScalaMap(m)
+        case _                                => value._1.toString -> value._2
+      }
+    }).toMap
+  }
+
+  /** Trait for implicit conversions for ConfigValue to native scala values */
   trait ImplicitConversions {
     import scala.language.implicitConversions
 
@@ -355,191 +314,115 @@ object ConfigUtils extends Logging {
       value != null && value.value != null && value.value != None