Config.scala 11.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
12 13 14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
15
package nl.lumc.sasc.biopet.utils.config
Peter van 't Hof's avatar
Peter van 't Hof committed
16

Peter van 't Hof's avatar
Peter van 't Hof committed
17
import java.io.{ File, PrintWriter }
Peter van 't Hof's avatar
Peter van 't Hof committed
18
import nl.lumc.sasc.biopet.utils.{ Logging, ConfigUtils }
19
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
20

21 22
/**
 * This class can store nested config values
23
 * @param _map Map with value for new config
24 25
 * @constructor Load config with existing map
 */
26
class Config(protected var _map: Map[String, Any],
Peter van 't Hof's avatar
Peter van 't Hof committed
27
             protected var _defaults: Map[String, Any] = Map()) extends Logging {
28
  logger.debug("Init phase of config")
29

Peter van 't Hof's avatar
Peter van 't Hof committed
30
  /** Default constructor */
Peter van 't Hof's avatar
Peter van 't Hof committed
31 32 33 34
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
35

36
  def map = _map
Peter van 't Hof's avatar
Peter van 't Hof committed
37
  def defaults = _defaults
38

39 40 41
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
Peter van 't Hof's avatar
Peter van 't Hof committed
42
   * @param default if true files are added to default instead of normal map
43
   */
44
  def loadConfigEnv(valueName: String, default: Boolean) {
Peter van 't Hof's avatar
Peter van 't Hof committed
45
    sys.env.get(valueName) match {
46
      case Some(globalFiles) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
47 48
        for (globalFile <- globalFiles.split(":")) {
          val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
49
          if (file.exists) {
Peter van 't Hof's avatar
Peter van 't Hof committed
50
            logger.info("Loading config file: " + file)
51
            loadConfigFile(file, default)
Peter van 't Hof's avatar
Peter van 't Hof committed
52 53 54
          } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
        }
      case _ => logger.info(valueName + " value not found, no global config is loaded")
55 56 57
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
58
  /** Loading default value for biopet */
59
  def loadDefaultConfig() {
60
    loadConfigEnv("BIOPET_CONFIG", default = true)
61 62
  }

63 64 65 66
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
67
  def loadConfigFile(configFile: File, default: Boolean = false) {
68
    val configMap = fileToConfigMap(configFile)
69
    if (default) {
Peter van 't Hof's avatar
Peter van 't Hof committed
70 71 72
      if (_defaults.isEmpty) _defaults = configMap
      else _defaults = mergeMaps(configMap, _defaults)
      logger.debug("New defaults: " + _defaults)
73
    } else {
74 75 76
      if (_map.isEmpty) _map = configMap
      else _map = mergeMaps(configMap, _map)
      logger.debug("New config: " + _map)
77
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
78
  }
bow's avatar
bow committed
79

80 81 82 83 84 85 86 87 88
  /**
   * Add a single vallue to the config
   * @param key key of value
   * @param value value itself
   * @param path Path to value
   * @param default if true value is put in default map
   */
  def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
    val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
Peter van 't Hof's avatar
Peter van 't Hof committed
89
    if (default) _defaults = mergeMaps(valueMap, _defaults)
90
    else _map = mergeMaps(valueMap, _map)
91 92
  }

93
  protected[config] var notFoundCache: Set[ConfigValueIndex] = Set()
94
  protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
95 96
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
97
  protected[config] def clearCache(): Unit = {
98
    notFoundCache = Set()
99 100 101
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
102

103 104 105 106 107 108
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
109
  def contains(s: String): Boolean = _map.contains(s)
110 111 112 113 114 115

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
116 117 118 119 120 121 122 123 124
  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @param fixedValues Fixed values
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
125
    if (notFoundCache.contains(requestedIndex)) false
126
    else if (fixedCache.contains(requestedIndex)) true
127
    else if (foundCache.contains(requestedIndex)) true
Peter van 't Hof's avatar
Peter van 't Hof committed
128
    else {
129 130 131
      val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
      if (fixedValue.isDefined) {
        fixedCache += (requestedIndex -> fixedValue.get)
132
        true
133
      } else {
134
        val value = Config.getValueFromMap(_map, requestedIndex)
135 136 137 138
        if (value.isDefined && value.get.value != None) {
          foundCache += (requestedIndex -> value.get)
          true
        } else {
139
          notFoundCache += requestedIndex
140 141
          false
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
142 143
      }
    }
144 145 146 147 148 149 150 151 152

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
153 154 155 156
  def contains(module: String, path: List[String],
               key: String,
               freeVar: Boolean = true,
               fixedValues: Map[String, Any] = Map()): Boolean = {
157
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
158
    contains(requestedIndex, fixedValues)
Peter van 't Hof's avatar
Peter van 't Hof committed
159
  }
bow's avatar
bow committed
160

161 162 163 164 165 166 167 168 169
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
170 171 172 173 174 175
  def apply(module: String,
            path: List[String],
            key: String,
            default: Any = null,
            freeVar: Boolean = true,
            fixedValues: Map[String, Any] = Map()): ConfigValue = {
176
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
177 178 179 180 181 182 183 184 185
    if (contains(requestedIndex, fixedValues)) {
      val fixedValue = fixedCache.get(requestedIndex)
      if (fixedValue.isDefined) {
        val userValue = Config.getValueFromMap(_map, requestedIndex)
        if (userValue.isDefined)
          logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
      }

      fixedValue.getOrElse(foundCache(requestedIndex))
Peter van 't Hof's avatar
Peter van 't Hof committed
186
    } else if (default != null) {
187
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
188 189
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
190
  }
bow's avatar
bow committed
191

Peter van 't Hof's avatar
Peter van 't Hof committed
192 193
  def writeReport(id: String, directory: File): Unit = {
    directory.mkdirs()
194 195 196 197 198 199 200 201 202 203 204 205 206 207

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
208
      val file = new File(directory, id + "." + name + ".json")
209 210 211 212 213 214 215
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
216
    val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
Peter van 't Hof's avatar
Peter van 't Hof committed
217
    val unused = uniqueKeys(map, found)
218 219 220 221 222 223 224 225 226 227

    def reportUnused(map: Map[String, Any], path: List[String] = Nil): Unit = {
      map.foreach {
        case (key, value: Map[_, _]) => reportUnused(value.asInstanceOf[Map[String, Any]], path :+ key)
        case (key, value) => logger.warn(s"config key in user config is never used, key: $key" +
          (if (path.nonEmpty) s", path: ${path.mkString(" -> ")}" else ""))
      }
    }

    reportUnused(unused)
228 229 230

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
231
    val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
232
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
233
    val notFound = convertIndexValuesToMap(notFoundCache.toList.map((_, None)), Some(false))
234 235 236 237 238

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

239 240
    writeMapToJsonFile(map, "input")
    writeMapToJsonFile(unused, "unused")
Peter van 't Hof's avatar
Peter van 't Hof committed
241
    writeMapToJsonFile(_defaults, "defaults")
242
    writeMapToJsonFile(found, "found")
243
    writeMapToJsonFile(fixed, "fixed")
244
    writeMapToJsonFile(effectiveFound, "effective.found")
245
    writeMapToJsonFile(effectiveFixed, "effective.fixed")
246 247 248 249 250 251
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

252
  override def toString: String = map.toString()
Peter van 't Hof's avatar
Peter van 't Hof committed
253 254
}

Peter van 't Hof's avatar
Peter van 't Hof committed
255
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
256 257
  val global = new Config

258 259 260
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
261
   * @param config2 Low prio map
262 263
   * @return Merged config
   */
264
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
Peter van 't Hof's avatar
Peter van 't Hof committed
265

266 267 268
  /**
   * Search for value in index position in a map
   * @param map Map to search in
269
   * @param startIndex Config index
270 271
   * @return Value
   */
272 273 274 275 276 277 278 279 280 281 282 283 284
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
285
      if (p.isDefined) p
286 287 288 289 290
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
291 292
      }
    }
293 294 295 296 297 298 299 300 301 302

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
303
    }
304 305 306 307 308 309 310 311

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

312
    tailSearch(startIndex.path)
313
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
314
}