Skip to content

GitLab

Commit c22acb87 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Config changed

parent cfee098a
Hide whitespace changes
Inline Side-by-side
Showing with 493 additions and 308 deletions
biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetCommandLineFunction.scala
View file @ c22acb87
......@@ -6,12 +6,14 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction
import org.broadinstitute.sting.commandline._
import scala.sys.process._
import scala.util.matching.Regex
import nl.lumc.sasc.biopet.core.config._
abstract class BiopetCommandLineFunction extends CommandLineFunction {
val globalConfig: Config
abstract class BiopetCommandLineFunction extends CommandLineFunction with Configurable {
// val globalConfig: Config
analysisName = getClass.getSimpleName
protected var config: Config = Config.mergeConfigs(globalConfig.getAsConfig(analysisName.toLowerCase), globalConfig)
logger.debug("config passed for " + analysisName)
// protected var config: Config = Config.mergeConfigs(globalConfig.getAsConfig(analysisName.toLowerCase), globalConfig)
// logger.debug("config passed for " + analysisName)
// logger.debug("config for " + analysisName + ": " + config)
@Input(doc="deps", required=false)
var deps: List[File] = Nil
......@@ -33,10 +35,10 @@ abstract class BiopetCommandLineFunction extends CommandLineFunction {
protected def afterGraph {
}
def setConfig(name:String) {
analysisName = name
config = Config.mergeConfigs(config.getAsConfig(analysisName.toLowerCase), config)
}
// def setConfig(name:String) {
// analysisName = name
// config = Config.mergeConfigs(config.getAsConfig(analysisName.toLowerCase), config)
// }
override def freezeFieldValues() {
checkExecuteble
......@@ -67,18 +69,18 @@ abstract class BiopetCommandLineFunction extends CommandLineFunction {
final protected def preCmdInternal {
checkExecuteble
//for (input <- this.inputs) if (!input.exists) throw new IllegalStateException("Input: " + input + " for " + analysisName + " is missing")
logger.debug("Config for " + analysisName + ": " + config)
//logger.debug("Config for " + analysisName + ": " + localConfig)
beforeCmd
addJobReportBinding("version", getVersion)
if (threads == 0) threads = config.getThreads(defaultThreads)
if (threads == 0) threads = getThreads(defaultThreads)
if (threads > 1) nCoresRequest = Option(threads)
addJobReportBinding("cores", if (nCoresRequest.get.toInt > 0) nCoresRequest.get.toInt else 1)
if (vmem == null) {
if (config.contains("vmem")) vmem = config.getAsString("vmem")
if (configContains("vmem")) vmem = config("vmem")
else if (!defaultVmem.isEmpty) vmem = defaultVmem
}
if (vmem != null) jobResourceRequests :+= "h_vmem=" + vmem
......
biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQCommandLine.scala
View file @ c22acb87
......@@ -109,7 +109,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
//temp :+= this.getClass.getResource(t.toString)
//logger.info(this.getClass.getResource(t.toString))
val s = if (t.getName.endsWith("/")) t.getName.substring(0, t.getName.length - 1) else t.getName
pipelineName = s + "." + System.currentTimeMillis
pipelineName = s.substring(0, s.lastIndexOf(".")) + "." + System.currentTimeMillis
}
new PluginManager[QScript](qPluginType, List(qScriptClasses.toURI.toURL))
}
......
biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala
View file @ c22acb87
......@@ -2,18 +2,20 @@ package nl.lumc.sasc.biopet.core
//import org.broadinstitute.sting.queue.QScript
import java.io.File
import org.broadinstitute.sting.queue.util.Logging
import nl.lumc.sasc.biopet.core.config._
abstract trait BiopetQScript extends Logging {
var config: Config = _
abstract trait BiopetQScript extends Configurable {
//var config: Config = _
var outputFiles:Map[String,File] = Map()
var samples:Map[String,Any] = Map()
def runSamplesJobs : Map[String,Map[String,File]] = {
final def runSamplesJobs : Map[String,Map[String,File]] = {
var output: Map[String,Map[String,File]] = Map()
if (config.contains("samples")) for ((key,value) <- config.getAsMap("samples")) {
var sample:Config = config.getAsConfig("samples").getAsConfig(key)
if (!sample.contains("ID")) sample.map += ("ID" -> key)
if (sample.getAsString("ID") == key) {
samples = config("samples")
if (globalConfig.contains("samples")) for ((key,value) <- samples) {
var sample = Configurable.any2map(value)
if (!sample.contains("ID")) sample += ("ID" -> key)
if (sample("ID") == key) {
var files:Map[String,List[File]] = runSingleSampleJobs(sample)
} else logger.warn("Key is not the same as ID on value for sample")
} else logger.warn("No Samples found in config")
......@@ -21,9 +23,9 @@ abstract trait BiopetQScript extends Logging {
}
def runSingleSampleJobs(sample:String) : Map[String,List[File]] ={
return runSingleSampleJobs(config.getAsConfig("samples").getAsConfig(sample))
return runSingleSampleJobs(Configurable.any2map(samples(sample)))
}
def runSingleSampleJobs(sampleConfig:Config) : Map[String,List[File]] = {
def runSingleSampleJobs(sampleConfig:Map[String,Any]) : Map[String,List[File]] = {
logger.debug("Default sample function, function 'runSingleSampleJobs' not defined in pipeline")
runRunsJobs(sampleConfig)
......@@ -31,19 +33,22 @@ abstract trait BiopetQScript extends Logging {
return Map()
}
def runRunsJobs(sampleConfig:Config) : Map[String,Map[String,File]] = {
final def runRunsJobs(sampleConfig:Map[String,Any]) : Map[String,Map[String,File]] = {
var output: Map[String,Map[String,File]] = Map()
val sampleID = sampleConfig.getAsString("ID")
if (sampleConfig.contains("runs")) for (key <- sampleConfig.getAsMap("runs").keySet) {
var run = sampleConfig.getAsConfig("runs").getAsConfig(key)
if (!run.contains("ID")) run.map += ("ID" -> key)
if (run.getAsString("ID") == key) {
output += key -> runSingleRunJobs(run, sampleConfig)
} else logger.warn("Key is not the same as ID on value for run of sample: " + sampleID)
val sampleID = sampleConfig("ID")
if (sampleConfig.contains("runs")) {
val runs = Configurable.any2map(sampleConfig("runs"))
for ((key,value) <- runs) {
var run = Configurable.any2map(value)
if (!run.contains("ID")) run += ("ID" -> key)
if (run("ID") == key) {
output += key -> runSingleRunJobs(run, sampleConfig)
} else logger.warn("Key is not the same as ID on value for run of sample: " + sampleID)
}
} else logger.warn("No runs found in config for sample: " + sampleID)
return output
}
def runSingleRunJobs(runConfig:Config, sampleConfig:Config) : Map[String,File] = {
def runSingleRunJobs(runConfig:Map[String,Any], sampleConfig:Map[String,Any]) : Map[String,File] = {
logger.debug("Default run function, function 'runSingleRunJobs' not defined in pipeline")
return Map()
......
biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/Config.scala deleted 100644 → 0
View file @ cfee098a
package nl.lumc.sasc.biopet.core
import scala.util.parsing.json._
import java.io.File
import org.broadinstitute.sting.queue.util.Logging
class Config(var map: Map[String,Any]) extends Logging {
def this() = {
this(Map())
logger.info("Init phase of config")
loadDefaultConfig()
}
def loadDefaultConfig() {
var globalFile: String = System.getenv("BIOPET_CONFIG")
if (globalFile != null) {
var file: File = new File(globalFile)
if (file.exists()) {
logger.info("Loading config file: " + file)
loadConfigFile(file)
}
else logger.warn("BIOPET_CONFIG value found but file does not exist, no global config is loaded")
} else logger.info("BIOPET_CONFIG value not found, no global config is loaded")
}
def contains(s:String) : Boolean = map.contains(s)
def loadConfigFile(configFile:File) {
var returnMap: Map[String,Any] = Map()
var configJson = JSON.parseFull(scala.io.Source.fromFile(configFile).mkString)
if (configJson == None) {
throw new IllegalStateException("The config JSON file is either not properly formatted or not a JSON file, file: " + configFile)
}
this.logger.debug("Jsonfile: " + configFile)
this.logger.debug("Contain: " + configJson)
configJson.get match {
case m:Map[_,_] => {
returnMap = Config.valueToMap(configJson.get)
if (map.isEmpty) returnMap = map
else map = Config.mergeMaps(returnMap, map)
}
case null => logger.warn("Config " + configFile + " wrong format")
}
this.logger.debug("config: " + map)
}
def getMap() : Map[String,Any] = map
def get(key:String) : Any = map(key)
def get(key:String, default:Any) : Any = if (contains(key)) get(key) else default
def getAsString(key:String) : String = map(key).toString
def getAsString(key:String, default:String) : String = if (contains(key)) getAsString(key) else default
def getAsInt(key:String) : Int = {
map(key) match {
case i:Double => return i.toInt
case i:Int => return i
case i:String => {
logger.warn("Value '" + key + "' is a string insteadof int in json file, trying auto convert")
return i.toInt
}
case _ => throw new IllegalStateException("Value '" + key + "' is not an int")
}
}
def getAsInt(s:String, default:Int) : Int = if (contains(s)) getAsInt(s) else default
def getAsDouble(key:String) : Double = {
map(key) match {
case d:Double => return d
case d:Int => return d.toDouble
case d:String => {
logger.warn("Value '" + key + "' is a string insteadof int in json file, trying auto convert")
return d.toDouble
}
case _ => throw new IllegalStateException("Value '" + key + "' is not an int")
}
}
def getAsDouble(key:String, default:Double) : Double = if (contains(key)) getAsDouble(key) else default
def getAsBoolean(key:String) : Boolean = {
map(key) match {
case b:Boolean => b
case b:String => {
logger.warn("Value '" + key + "' is a string insteadof boolean in json file, trying auto convert")
return b.contains("true")
}
case b:Int => {
logger.warn("Value '" + key + "' is a int insteadof boolean in json file, trying auto convert")
(b > 0)
}
case _ => throw new IllegalStateException("Value '" + key + "' is not an boolean")
}
}
def getAsBoolean(key:String, default:Boolean) : Boolean = if (contains(key)) getAsBoolean(key) else default
def getAsList(key:String) : List[Any] = {
map(key) match {
case l:List[_] => return l
case s:String => return List(s)
case _ => throw new IllegalStateException("Value '" + key + "' is not an List")
}
}
def getAsList(key:String, default:List[Any]) : List[Any] = if (contains(key)) getAsList(key) else default
def getAsListOfStrings(key:String) : List[String] = {
var l: List[String] = Nil
for (v <- getAsList(key)) l :+= v.toString
return l
}
def getAsListOfStrings(key:String, default:List[String]) : List[String] = if (contains(key)) getAsListOfStrings(key) else default
def getAsMap(key:String) : Map[String,Any] = {
map(key) match {
case m:Map[_,_] => return Config.valueToMap(m)
case _ => throw new IllegalStateException("Value '" + key + "' is not an Map")
}
}
def getAsMap(key:String, default:Map[String,Any]) : Map[String,Any] = if (contains(key)) getAsMap(key) else default
def getAsConfig(key:String, default:Map[String,Any]) : Config = if (contains(key)) new Config(getAsMap(key)) else new Config(default)
def getAsConfig(key:String, default:Config) : Config = if (contains(key)) Config.mergeConfigs(getAsConfig(key), default) else default
def getAsConfig(key:String, default:Config, subDefault:String) : Config = {
if (contains(key)) Config.mergeConfigs(getAsConfig(key), default.getAsConfig(subDefault))
else default
}
def getAsConfig(s:String) : Config = if (contains(s)) new Config(getAsMap(s)) else new Config(Map())
def getThreads(default:Int) : Int = {
val maxThreads = this.getAsInt("maxthreads", 8)
val threads = this.getAsInt("threads", default)
if (maxThreads > threads) return threads
else return maxThreads
}
override def toString() : String = map.toString
}
object Config {
def valueToMap(input:Any) : Map[String,Any] = {
var ouputMap: Map[String,Any] = Map()
input match {
case m:Map[_, _] => {
for ((k,v) <- m) {
k match {
case s:String => ouputMap += (s -> v)
case _ => throw new IllegalStateException("Key of map '" + m + "' is not an String")
}
}
}
case _ => throw new IllegalStateException("Value '" + input + "' is not an Map")
}
return ouputMap
}
def mergeMaps(map1:Map[String,Any],map2:Map[String,Any]) : Map[String,Any] = {
var newMap: Map[String,Any] = Map()
for (key <- map1.keySet.++(map2.keySet)) {
if (map1.contains(key) && !map2.contains(key)) newMap += (key -> map1(key))
else if (!map1.contains(key) && map2.contains(key)) newMap += (key -> map2(key))
else if (map1.contains(key) && map2.contains(key)) {
map1(key) match {
case m1:Map[_,_] => {
map2(key) match {
case m2:Map[_,_] => newMap += (key -> mergeMaps(Config.valueToMap(m1),Config.valueToMap(m2)))
case _ => newMap += (key -> map1(key))
}
}
case _ => newMap += (key -> map1(key))
}
}
}
return newMap
}
def mergeConfigs(config1:Config,config2:Config) : Config = new Config(mergeMaps(config1.getMap, config2.getMap))
}
\ No newline at end of file
biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/config/Config.scala 0 → 100644
View file @ c22acb87
package nl.lumc.sasc.biopet.core.config
import nl.lumc.sasc.biopet.core._
import scala.util.parsing.json._
import java.io.File
import org.broadinstitute.sting.queue.util.Logging
class Config(var map: Map[String,Any]) extends Logging {
def this() = {
this(Map())
logger.info("Init phase of config")
loadDefaultConfig()
}
def loadDefaultConfig() {
var globalFile: String = System.getenv("BIOPET_CONFIG")
if (globalFile != null) {
var file: File = new File(globalFile)
if (file.exists()) {
logger.info("Loading config file: " + file)
loadConfigFile(file)
}
else logger.warn("BIOPET_CONFIG value found but file does not exist, no global config is loaded")
} else logger.info("BIOPET_CONFIG value not found, no global config is loaded")
}
def loadConfigFile(configFile:File) {
var configJson = JSON.parseFull(scala.io.Source.fromFile(configFile).mkString)
if (configJson == None) {
throw new IllegalStateException("The config JSON file is either not properly formatted or not a JSON file, file: " + configFile)
}
this.logger.debug("Jsonfile: " + configFile)
this.logger.debug("Contain: " + configJson)
configJson.get match {
case m:Map[_,_] => {
logger.debug(m)
if (map.isEmpty) map = m.asInstanceOf[Map[String,Any]]
else map = Config.mergeMaps(m.asInstanceOf[Map[String,Any]], map)
}
case null => logger.warn("Config " + configFile + " wrong format")
}
this.logger.debug("config: " + map)
}
def getMap() : Map[String,Any] = map
var notFoundCache: List[ConfigValueIndex] = List()
var foundCache: Map[ConfigValueIndex,ConfigValue] = Map()
def contains(s:String) : Boolean = map.contains(s)
def contains(requestedIndex:ConfigValueIndex) : Boolean = contains(requestedIndex.module, requestedIndex.path, requestedIndex.key)
def contains(module:String, path: List[String], key:String) : Boolean = {
val requestedIndex = ConfigValueIndex(module,path,key)
if (notFoundCache.contains(requestedIndex)) return false
else if (foundCache.contains(requestedIndex)) return true
else {
var submodules = path.reverse
while (!submodules.isEmpty) {
var submodules2 = submodules
while (!submodules2.isEmpty) {
val p = getMapFromPath(submodules2 ::: module :: Nil)
//logger.debug("p: " + p)
if (p.contains(key)) {
foundCache += (requestedIndex -> ConfigValue.apply(requestedIndex, ConfigValueIndex(module,submodules2,key), p(key)))
return true
}
val p2 = getMapFromPath(submodules2)
//logger.debug("p2: " + p2)
if (p2.contains(key)) {
foundCache += (requestedIndex -> ConfigValue.apply(requestedIndex, ConfigValueIndex(module,submodules2,key), p2(key)))
return true
}
submodules2 = submodules2.init
}
submodules = submodules.tail
}
val p = getMapFromPath(module :: Nil)
if (p.contains(key)) { // Module is not nested
foundCache += (requestedIndex -> ConfigValue.apply(requestedIndex, ConfigValueIndex(module,Nil,key), p(key)))
return true
} else if (this.contains(key)) { // Root value of json
foundCache += (requestedIndex -> ConfigValue.apply(requestedIndex, ConfigValueIndex("",Nil,key), get(key)))
return true
} else { // At this point key is not found on the path
notFoundCache +:= requestedIndex
return false
}
}
}
private def get(key:String) : Any = map(key)
private def get(key:String, default:Any) : Any = if (contains(key)) get(key) else default
def apply(module:String, path: List[String], key:String, default:Any) : ConfigValue = {
val requestedIndex = ConfigValueIndex(module,path,key)
if (contains(requestedIndex)) return foundCache(requestedIndex)
else {
foundCache += (requestedIndex -> ConfigValue.apply(requestedIndex, null, default))
return foundCache(requestedIndex)
}
}
def apply(module:String, path: List[String], key:String) : ConfigValue = {
val requestedIndex = ConfigValueIndex(module,path,key)
if (contains(requestedIndex)) return foundCache(requestedIndex)
else {
logger.error("Value in config could not be found but it seems required, inde: " + requestedIndex)
throw new IllegalStateException("Value in config could not be found but it seems required, index: " + requestedIndex)
}
}
private def getMapFromPath(path: List[String]) : Map[String,Any] = {
var returnMap: Map[String,Any] = map
for (m <- path) {
if (!returnMap.contains(m)) return Map()
else returnMap = Config.valueToMap(returnMap(m))
}
return returnMap
}
private def check(module:String, path: List[String], key:String) {
}
// def getAsString(key:String) : String = map(key).toString
// def getAsString(key:String, default:String) : String = if (contains(key)) getAsString(key) else default
//
// def getAsInt(key:String) : Int = {
// map(key) match {
// case i:Double => return i.toInt
// case i:Int => return i
// case i:String => {
// logger.warn("Value '" + key + "' is a string insteadof int in json file, trying auto convert")
// return i.toInt
// }
// case _ => throw new IllegalStateException("Value '" + key + "' is not an int")
// }
// }
// def getAsInt(s:String, default:Int) : Int = if (contains(s)) getAsInt(s) else default
//
// def getAsDouble(key:String) : Double = {
// map(key) match {
// case d:Double => return d
// case d:Int => return d.toDouble
// case d:String => {
// logger.warn("Value '" + key + "' is a string insteadof int in json file, trying auto convert")
// return d.toDouble
// }
// case _ => throw new IllegalStateException("Value '" + key + "' is not an int")
// }
// }
// def getAsDouble(key:String, default:Double) : Double = if (contains(key)) getAsDouble(key) else default
//
// def getAsBoolean(key:String) : Boolean = {
// map(key) match {
// case b:Boolean => b
// case b:String => {
// logger.warn("Value '" + key + "' is a string insteadof boolean in json file, trying auto convert")
// return b.contains("true")
// }
// case b:Int => {
// logger.warn("Value '" + key + "' is a int insteadof boolean in json file, trying auto convert")
// (b > 0)
// }
// case _ => throw new IllegalStateException("Value '" + key + "' is not an boolean")
// }
// }
// def getAsBoolean(key:String, default:Boolean) : Boolean = if (contains(key)) getAsBoolean(key) else default
//
// def getAsList(key:String) : List[Any] = {
// map(key) match {
// case l:List[_] => return l
// case s:String => return List(s)
// case _ => throw new IllegalStateException("Value '" + key + "' is not an List")
// }
// }
// def getAsList(key:String, default:List[Any]) : List[Any] = if (contains(key)) getAsList(key) else default
// def getAsListOfStrings(key:String) : List[String] = {
// var l: List[String] = Nil
// for (v <- getAsList(key)) l :+= v.toString
// return l
// }
// def getAsListOfStrings(key:String, default:List[String]) : List[String] = if (contains(key)) getAsListOfStrings(key) else default
//
// def getAsMap(key:String) : Map[String,Any] = Config.valueToMap(map(key))
// def getAsMap(key:String, default:Map[String,Any]) : Map[String,Any] = if (contains(key)) getAsMap(key) else default
//
// def getAsConfig(key:String, default:Map[String,Any]) : Config = if (contains(key)) new Config(getAsMap(key)) else new Config(default)
// def getAsConfig(key:String, default:Config) : Config = if (contains(key)) Config.mergeConfigs(getAsConfig(key), default) else default
// def getAsConfig(key:String, default:Config, subDefault:String) : Config = {
// if (contains(key)) Config.mergeConfigs(getAsConfig(key), default.getAsConfig(subDefault))
// else default
// }
// def getAsConfig(s:String) : Config = if (contains(s)) new Config(getAsMap(s)) else new Config(Map())
//
// def getThreads(default:Int) : Int = {
// val maxThreads = this.getAsInt("maxthreads", 8)
// val threads = this.getAsInt("threads", default)
// if (maxThreads > threads) return threads
// else return maxThreads
// }
override def toString() : String = map.toString
}
object Config {
def valueToMap(input:Any) : Map[String,Any] = {
input match {
case m:Map[_, _] => return m.asInstanceOf[Map[String,Any]]
case _ => throw new IllegalStateException("Value '" + input + "' is not an Map")
}
}
def mergeMaps(map1:Map[String,Any],map2:Map[String,Any]) : Map[String,Any] = {
var newMap: Map[String,Any] = Map()
for (key <- map1.keySet.++(map2.keySet)) {
if (map1.contains(key) && !map2.contains(key)) newMap += (key -> map1(key))
else if (!map1.contains(key) && map2.contains(key)) newMap += (key -> map2(key))
else if (map1.contains(key) && map2.contains(key)) {
map1(key) match {
case m1:Map[_,_] => {
map2(key) match {
case m2:Map[_,_] => newMap += (key -> mergeMaps(Config.valueToMap(m1),Config.valueToMap(m2)))
case _ => newMap += (key -> map1(key))