Commit e35cb525 authored by Wai Yi Leung's avatar Wai Yi Leung
Browse files

Merge branch 'feature-yaml_support' into 'develop'

Feature yaml support

Issue #142

Now implemented a yaml parser and commandline argument to add values to config. Json still uses the old parser, when a file extension is ".yaml" the new parser is used. Later we might remove the old parser completely since SnakeYaml also supports json.

See merge request !146
parents 3fbdc514 11c99934
......@@ -110,5 +110,10 @@
<artifactId>scopt_2.10</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.15</version>
</dependency>
</dependencies>
</project>
......@@ -25,19 +25,19 @@ import org.broadinstitute.gatk.queue.function.scattergather.ScatterGatherableFun
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
import scala.collection.mutable.ListBuffer
/**
* Base for biopet pipeline
*/
/** Base for biopet pipeline */
trait BiopetQScript extends Configurable with GatkLogging {
@Argument(doc = "JSON config file(s)", fullName = "config_file", shortName = "config", required = false)
@Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
val configfiles: List[File] = Nil
@Argument(doc = "Config values, value should be formatted like 'key=value' or 'path:path:key=value'", fullName = "config_value", shortName = "cv", required = false)
val configValues: List[String] = Nil
/** Output directory of pipeline */
var outputDir: File = {
Config.getValueFromMap(globalConfig.map, ConfigValueIndex(this.configName, configPath, "output_dir")) match {
case Some(value) => new File(value.asString).getAbsoluteFile
case _ => new File(".")
}
if (config.contains("output_dir", path = Nil)) config("output_dir", path = Nil).asFile
else new File(".")
}
@Argument(doc = "Disable all scatters", shortName = "DSC", required = false)
......@@ -57,11 +57,10 @@ trait BiopetQScript extends Configurable with GatkLogging {
/** Pipeline itself */
def biopetScript
/**
* Script from queue itself, final to force some checks for each pipeline and write report
*/
/** Script from queue itself, final to force some checks for each pipeline and write report */
final def script() {
outputDir = config("output_dir").asFile.getAbsoluteFile
outputDir = config("output_dir")
outputDir = outputDir.getAbsoluteFile
init
biopetScript
......
......@@ -20,9 +20,7 @@ import java.io.File
import nl.lumc.sasc.biopet.core.config.Config
import nl.lumc.sasc.biopet.core.workaround.BiopetQCommandLine
/**
* Wrapper around executable from Queue
*/
/** Wrapper around executable from Queue */
trait PipelineCommand extends MainCommand with GatkLogging {
/**
......@@ -31,10 +29,7 @@ trait PipelineCommand extends MainCommand with GatkLogging {
*/
def pipeline = "/" + getClass.getName.stripSuffix("$").replaceAll("\\.", "/") + ".class"
/**
* Class can be used directly from java with -cp option
* @param args
*/
/** Class can be used directly from java with -cp option */
def main(args: Array[String]): Unit = {
val argsSize = args.size
for (t <- 0 until argsSize) {
......@@ -42,6 +37,17 @@ trait PipelineCommand extends MainCommand with GatkLogging {
if (t >= argsSize) throw new IllegalStateException("-config needs a value")
Config.global.loadConfigFile(new File(args(t + 1)))
}
if (args(t) == "-cv" || args(t) == "--config_value") {
val v = args(t + 1).split("=")
require(v.size == 2, "Value should be formatted like 'key=value' or 'path:path:key=value'")
val value = v(1)
val p = v(0).split(":")
val key = p.last
val path = p.dropRight(1).toList
Config.global.addValue(key, value, path)
}
if (args(t) == "--logging_level" || args(t) == "-l") {
args(t + 1).toLowerCase match {
case "debug" => Logging.logger.setLevel(org.apache.log4j.Level.DEBUG)
......
......@@ -20,19 +20,16 @@ import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.utils.ConfigUtils
import nl.lumc.sasc.biopet.utils.ConfigUtils._
import scala.reflect.io.Directory
/**
* This class can store nested config values
* @param map Map with value for new config
* @constructor Load config with existing map
*/
class Config(var map: Map[String, Any]) extends Logging {
class Config(var map: Map[String, Any],
protected[core] var defaults: Map[String, Any] = Map()) extends Logging {
logger.debug("Init phase of config")
/**
* Default constructor
*/
/** Default constructor */
def this() = {
this(Map())
loadDefaultConfig()
......@@ -41,15 +38,16 @@ class Config(var map: Map[String, Any]) extends Logging {
/**
* Loading a environmental variable as location of config files to merge into the config
* @param valueName Name of value
* @param default if true files are added to default instead of normal map
*/
def loadConfigEnv(valueName: String) {
def loadConfigEnv(valueName: String, default: Boolean) {
sys.env.get(valueName) match {
case Some(globalFiles) => {
for (globalFile <- globalFiles.split(":")) {
val file: File = new File(globalFile)
if (file.exists) {
logger.info("Loading config file: " + file)
loadConfigFile(file)
loadConfigFile(file, default)
} else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
}
}
......@@ -57,23 +55,39 @@ class Config(var map: Map[String, Any]) extends Logging {
}
}
/**
* Loading default value for biopet
*/
/** Loading default value for biopet */
def loadDefaultConfig() {
loadConfigEnv("BIOPET_CONFIG")
loadConfigEnv("BIOPET_CONFIG", true)
}
/**
* Merge a json file into the config
* @param configFile Location of file
*/
def loadConfigFile(configFile: File) {
def loadConfigFile(configFile: File, default: Boolean = false) {
val configMap = fileToConfigMap(configFile)
if (default) {
if (defaults.isEmpty) defaults = configMap
else defaults = mergeMaps(configMap, defaults)
logger.debug("New defaults: " + defaults)
} else {
if (map.isEmpty) map = configMap
else map = mergeMaps(configMap, map)
logger.debug("New config: " + map)
}
}
if (map.isEmpty) map = configMap
else map = mergeMaps(configMap, map)
logger.debug("New config: " + map)
/**
* Add a single vallue to the config
* @param key key of value
* @param value value itself
* @param path Path to value
* @param default if true value is put in default map
*/
def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
if (default) defaults = mergeMaps(valueMap, defaults)
else map = mergeMaps(valueMap, map)
}
protected[config] var notFoundCache: List[ConfigValueIndex] = List()
......
......@@ -19,64 +19,34 @@ import java.io.File
import nl.lumc.sasc.biopet.utils.ConfigUtils._
class ConfigValue(val requestIndex: ConfigValueIndex, val foundIndex: ConfigValueIndex, val value: Any, val default: Boolean) {
/**
* Get value as String
* @return value as String
*/
/** Get value as String */
def asString = any2string(value)
/**
* Get value as File
* @return value as File
*/
/** Get value as File */
def asFile = new File(any2string(value))
/**
* Get value as Int
* @return value as Int
*/
/** Get value as Int */
def asInt = any2int(value)
/**
* Get value as Double
* @return value as Double
*/
/** Get value as Double */
def asDouble = any2double(value)
/**
* Get value as List[Any]
* @return value as List[Any]
*/
/** Get value as List[Any] */
def asList = any2list(value)
/**
* Get value as List[File]
* @return value as List[File]
*/
/** Get value as List[File] */
def asFileList: List[File] = for (file <- any2stringList(value)) yield new File(file)
/**
* Get value as List[String]
* @return value as List[String]
*/
/** Get value as List[String] */
def asStringList: List[String] = any2stringList(value)
/**
* Get value as Map
* @return value as Map
*/
/** Get value as Map */
def asMap = any2map(value)
/**
* Get value as Boolean
* @return value as Boolean
*/
/** Get value as Boolean */
def asBoolean = any2boolean(value)
/**
* Readable output of indexes and value, just for debug
* @return
*/
/** Readable output of indexes and value, just for debug */
override def toString: String = {
var output = "key = " + requestIndex.key
output += ", value = " + value
......
......@@ -15,9 +15,7 @@
*/
package nl.lumc.sasc.biopet.core.config
import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
import scala.collection.JavaConversions._
trait Configurable extends ImplicitConversions {
/** Should be object of parant object */
......@@ -39,7 +37,7 @@ trait Configurable extends ImplicitConversions {
/** Map to store defaults for config */
def defaults: Map[String, Any] = {
if (root != null) root.defaults
else Map()
else globalConfig.defaults
}
val config = new ConfigFunctions
......@@ -52,15 +50,13 @@ trait Configurable extends ImplicitConversions {
* @param submodule
* @return
*/
def path(sample: String = null, library: String = null, submodule: String = null) = {
def getConfigPath(sample: String = null, library: String = null, submodule: String = null) = {
(if (sample != null) "samples" :: sample :: Nil else Nil) :::
(if (library != null) "libraries" :: library :: Nil else Nil) :::
(if (submodule != null) configPath ::: configName :: Nil else configPath)
}
/**
* Class is used for retrieval of config values
*/
/** Class is used for retrieval of config values */
protected class ConfigFunctions(val defaultSample: Option[String] = None, val defaultLibrary: Option[String] = None) {
def this(defaultSample: String, defaultLibrary: String) = {
this(defaultSample = Some(defaultSample), defaultLibrary = Some(defaultLibrary))
......@@ -91,11 +87,12 @@ trait Configurable extends ImplicitConversions {
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null): ConfigValue = {
library: String = null,
path: List[String] = null): ConfigValue = {
val s = if (sample != null || defaultSample.isEmpty) sample else defaultSample.get
val l = if (library != null || defaultLibrary.isEmpty) library else defaultLibrary.get
val m = if (submodule != null) submodule else configName
val p = path(s, l, submodule)
val p = if (path == null) getConfigPath(s, l, submodule) else path
val d = {
val value = Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar))
if (value.isDefined) value.get.value else default
......@@ -117,11 +114,12 @@ trait Configurable extends ImplicitConversions {
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null) = {
library: String = null,
path: List[String] = null) = {
val s = if (sample != null || defaultSample.isEmpty) sample else defaultSample.get
val l = if (library != null || defaultLibrary.isEmpty) library else defaultLibrary.get
val m = if (submodule != null) submodule else configName
val p = path(s, l, submodule)
val p = if (path == null) getConfigPath(s, l, submodule) else path
globalConfig.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
}
......
......@@ -20,7 +20,9 @@ import nl.lumc.sasc.biopet.core.BiopetQScript
import nl.lumc.sasc.biopet.core.Logging
import nl.lumc.sasc.biopet.core.config.ConfigValue
import argonaut._, Argonaut._
import org.yaml.snakeyaml.Yaml
import scalaz._, Scalaz._
import scala.collection.JavaConversions._
/**
* This object contains general function for the config
......@@ -78,16 +80,13 @@ object ConfigUtils extends Logging {
val value = map.get(path.head)
if (path.tail == Nil || value == None) value
else value.get match {
case map: Map[_, _] => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
case _ => None
case map: Map[_, _] => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
case map: java.util.LinkedHashMap[_, _] => getValueFromPath(map.toMap.asInstanceOf[Map[String, Any]], path.tail)
case _ => None
}
}
/**
* Make json aboject from a file
* @param configFile Input file
* @return Json object
*/
/** Make json aboject from a file */
def fileToJson(configFile: File): Json = {
logger.debug("Jsonfile: " + configFile)
val jsonText = scala.io.Source.fromFile(configFile).mkString
......@@ -99,22 +98,25 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert config value to map
* @param configFile
* @return Config map
*/
/** Convert config value to map */
def fileToConfigMap(configFile: File): Map[String, Any] = {
val configJson = jsonToMap(fileToJson(configFile))
logger.debug("Contain: " + configJson)
return configJson
val configMap = {
if (configFile.getName.endsWith(".yaml")) yamlToMap(configFile)
else jsonToMap(fileToJson(configFile))
}
logger.debug("Contain: " + configMap)
return configMap
}
/**
* Convert json to native scala map/values
* @param json input json
* @return
*/
/** Convert a yaml file to map[String, Any] */
def yamlToMap(file: File): Map[String, Any] = {
val yaml = new Yaml()
val a = yaml.load(scala.io.Source.fromFile(file).reader())
ConfigUtils.any2map(a)
}
/** Convert json to native scala map/values */
def jsonToMap(json: Json): Map[String, Any] = {
var output: Map[String, Any] = Map()
if (json.isObject) {
......@@ -126,11 +128,7 @@ object ConfigUtils extends Logging {
return output
}
/**
* Convert json value to native scala value
* @param json input json
* @return
*/
/** Convert json value to native scala value */
def jsonToAny(json: Json): Any = {
if (json.isObject) return jsonToMap(json)
else if (json.isArray) {
......@@ -147,11 +145,7 @@ object ConfigUtils extends Logging {
else throw new IllegalStateException("Config value type not supported, value: " + json)
}
/**
* Convert native scala map to json
* @param map Input map
* @return
*/
/** Convert native scala map to json */
def mapToJson(map: Map[String, Any]): Json = {
map.foldLeft(jEmptyObject)((acc, kv) => (kv._1 := {
kv._2 match {
......@@ -161,11 +155,7 @@ object ConfigUtils extends Logging {
}) ->: acc)
}
/**
* Convert native scala value to json, fall back on .toString if type is not a native scala value
* @param any Input Any value
* @return
*/
/** Convert native scala value to json, fall back on .toString if type is not a native scala value */
def anyToJson(any: Any): Json = {
any match {
case j: Json => j
......@@ -183,11 +173,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to String
* @param any Input Any value
* @return
*/
/** Convert Any to String */
def any2string(any: Any): String = {
if (any == null) return null
any match {
......@@ -196,11 +182,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to Int
* @param any Input Any value
* @return
*/
/** Convert Any to Int */
def any2int(any: Any): Int = {
any match {
case i: Int => i
......@@ -214,11 +196,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to Long
* @param any Input Any value
* @return
*/
/** Convert Any to Long */
def any2long(any: Any): Long = {
any match {
case l: Double => l.toLong
......@@ -232,11 +210,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to Double
* @param any Input Any value
* @return
*/
/** Convert Any to Double */
def any2double(any: Any): Double = {
any match {
case d: Double => d
......@@ -251,11 +225,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to Float
* @param any Input Any value
* @return
*/
/** Convert Any to Float */
def any2float(any: Any): Float = {
any match {
case f: Double => f.toFloat
......@@ -270,11 +240,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to Boolean
* @param any Input Any value
* @return
*/
/** Convert Any to Boolean */
def any2boolean(any: Any): Boolean = {
any match {
case b: Boolean => b
......@@ -290,11 +256,7 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to List[Any], fallback on list with 1 value
* @param any Input Any value
* @return
*/
/** Convert Any to List[Any], fallback on list with 1 value */
def any2list(any: Any): List[Any] = {
if (any == null) return null
any match {
......@@ -303,42 +265,39 @@ object ConfigUtils extends Logging {
}
}
/**
* Convert Any to List[String]
* @param any Input Any value
* @return
*/
/** Convert Any to List[String] */
def any2stringList(any: Any): List[String] = {
if (any == null) return null
any2list(any).map(_.toString)
}
/**
* Convert Any to List[File]
* @param any Input Any value
* @return
*/
/** Convert Any to List[File] */
def any2fileList(any: Any): List[File] = {
if (any == null) return null
any2list(any).map(x => new File(x.toString))
}
/**
* Convert Any to Map[String, Any]
* @param any Input Any value
* @return
*/
/** Convert Any to Map[String, Any] */
def any2map(any: Any): Map[String, Any] = {
if (any == null) return null
any match {
case m: Map[_, _] => m.map(x => x._1.toString -> x._2)
case _ => throw new IllegalStateException("Value '" + any + "' is not an Map")
case m: Map[_, _] => m.map(x => x._1.toString -> x._2)
case m: java.util.LinkedHashMap[_, _] => nestedJavaHashMaptoScalaMap(m)
case _ => throw new IllegalStateException("Value '" + any + "' is not an Map")
}
}
/**
* Trait for implicit conversions for ConfigValue to native scala values
*/
/** Convert nested java hash map to scala hash map */
def nestedJavaHashMaptoScalaMap(input: java.util.LinkedHashMap[_, _]): Map[String, Any] = {
input.map(value => {
value._2 match {
case m: java.util.LinkedHashMap[_, _] => value._1.toString -> nestedJavaHashMaptoScalaMap(m)
case _ => value._1.toString -> value._2
}
}).toMap
}
/** Trait for implicit conversions for ConfigValue to native scala values */
trait ImplicitConversions {
import scala.language.implicitConversions
......@@ -355,191 +314,115 @@ object ConfigUtils extends Logging {
value != null && value.value != null && value.value != None