Config.scala 11.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
12 13 14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
15
package nl.lumc.sasc.biopet.utils.config
Peter van 't Hof's avatar
Peter van 't Hof committed
16

Peter van 't Hof's avatar
Peter van 't Hof committed
17
import java.io.{ File, PrintWriter }
18 19

import nl.lumc.sasc.biopet.utils.{ConfigUtils, Logging}
20
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
21

22 23 24
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

25 26
/**
 * This class can store nested config values
27
 * @param _map Map with value for new config
28 29
 * @constructor Load config with existing map
 */
30
class Config(protected var _map: Map[String, Any],
Peter van 't Hof's avatar
Peter van 't Hof committed
31
             protected var _defaults: Map[String, Any] = Map()) extends Logging {
32
  logger.debug("Init phase of config")
33

Peter van 't Hof's avatar
Peter van 't Hof committed
34
  /** Default constructor */
Peter van 't Hof's avatar
Peter van 't Hof committed
35 36 37 38
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
39

40
  def map = _map
Peter van 't Hof's avatar
Peter van 't Hof committed
41
  def defaults = _defaults
42

43 44 45
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
Peter van 't Hof's avatar
Peter van 't Hof committed
46
   * @param default if true files are added to default instead of normal map
47
   */
48
  def loadConfigEnv(valueName: String, default: Boolean) {
Peter van 't Hof's avatar
Peter van 't Hof committed
49
    sys.env.get(valueName) match {
50
      case Some(globalFiles) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
51 52
        for (globalFile <- globalFiles.split(":")) {
          val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
53
          if (file.exists) {
54
            logger.debug("Loading config file: " + file)
55
            loadConfigFile(file, default)
Peter van 't Hof's avatar
Peter van 't Hof committed
56 57
          } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
        }
58
      case _ => logger.debug(valueName + " value not found, no global config is loaded")
59 60 61
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
62
  /** Loading default value for biopet */
63
  def loadDefaultConfig() {
64
    loadConfigEnv("BIOPET_CONFIG", default = true)
65 66
  }

67 68 69 70
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
71
  def loadConfigFile(configFile: File, default: Boolean = false) {
72
    val configMap = fileToConfigMap(configFile)
73
    if (default) {
Peter van 't Hof's avatar
Peter van 't Hof committed
74 75 76
      if (_defaults.isEmpty) _defaults = configMap
      else _defaults = mergeMaps(configMap, _defaults)
      logger.debug("New defaults: " + _defaults)
77
    } else {
78 79 80
      if (_map.isEmpty) _map = configMap
      else _map = mergeMaps(configMap, _map)
      logger.debug("New config: " + _map)
81
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
82
  }
bow's avatar
bow committed
83

84 85 86 87 88 89 90 91 92
  /**
   * Add a single vallue to the config
   * @param key key of value
   * @param value value itself
   * @param path Path to value
   * @param default if true value is put in default map
   */
  def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
    val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
Peter van 't Hof's avatar
Peter van 't Hof committed
93
    if (default) _defaults = mergeMaps(valueMap, _defaults)
94
    else _map = mergeMaps(valueMap, _map)
95 96
  }

97
  protected[config] var notFoundCache: Set[ConfigValueIndex] = Set()
98
  protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
99 100
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
101
  protected[config] def clearCache(): Unit = {
102
    notFoundCache = Set()
103 104 105
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
106

107 108 109 110 111 112
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
113
  def contains(s: String): Boolean = _map.contains(s)
114 115 116 117 118 119

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
120 121 122 123 124 125 126 127 128
  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @param fixedValues Fixed values
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
129
    if (notFoundCache.contains(requestedIndex)) false
130
    else if (fixedCache.contains(requestedIndex)) true
131
    else if (foundCache.contains(requestedIndex)) true
Peter van 't Hof's avatar
Peter van 't Hof committed
132
    else {
133 134 135
      val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
      if (fixedValue.isDefined) {
        fixedCache += (requestedIndex -> fixedValue.get)
136
        true
137
      } else {
138
        val value = Config.getValueFromMap(_map, requestedIndex)
139 140 141 142
        if (value.isDefined && value.get.value != None) {
          foundCache += (requestedIndex -> value.get)
          true
        } else {
143
          notFoundCache += requestedIndex
144 145
          false
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
146 147
      }
    }
148 149 150 151 152 153 154 155 156

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
157 158 159 160
  def contains(module: String, path: List[String],
               key: String,
               freeVar: Boolean = true,
               fixedValues: Map[String, Any] = Map()): Boolean = {
161
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
162
    contains(requestedIndex, fixedValues)
Peter van 't Hof's avatar
Peter van 't Hof committed
163
  }
bow's avatar
bow committed
164

165 166 167 168 169 170 171 172 173
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
174 175 176 177 178 179
  def apply(module: String,
            path: List[String],
            key: String,
            default: Any = null,
            freeVar: Boolean = true,
            fixedValues: Map[String, Any] = Map()): ConfigValue = {
180
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
181 182 183 184 185 186 187 188 189
    if (contains(requestedIndex, fixedValues)) {
      val fixedValue = fixedCache.get(requestedIndex)
      if (fixedValue.isDefined) {
        val userValue = Config.getValueFromMap(_map, requestedIndex)
        if (userValue.isDefined)
          logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
      }

      fixedValue.getOrElse(foundCache(requestedIndex))
Peter van 't Hof's avatar
Peter van 't Hof committed
190
    } else if (default != null) {
191
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
192 193
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
194
  }
bow's avatar
bow committed
195

196
  def writeReport(directory: File): Future[Unit] = Future {
Peter van 't Hof's avatar
Peter van 't Hof committed
197
    directory.mkdirs()
198 199 200 201 202 203 204 205 206 207 208 209 210 211

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
212
      val file = new File(directory, name + ".json")
213 214 215 216 217 218 219
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
220
    val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
Peter van 't Hof's avatar
Peter van 't Hof committed
221
    val unused = uniqueKeys(map, found)
222 223 224 225 226 227 228 229 230 231

    def reportUnused(map: Map[String, Any], path: List[String] = Nil): Unit = {
      map.foreach {
        case (key, value: Map[_, _]) => reportUnused(value.asInstanceOf[Map[String, Any]], path :+ key)
        case (key, value) => logger.warn(s"config key in user config is never used, key: $key" +
          (if (path.nonEmpty) s", path: ${path.mkString(" -> ")}" else ""))
      }
    }

    reportUnused(unused)
232 233 234

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
235
    val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
236
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
237
    val notFound = convertIndexValuesToMap(notFoundCache.toList.map((_, None)), Some(false))
238 239 240 241 242

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

243 244
    writeMapToJsonFile(map, "input")
    writeMapToJsonFile(unused, "unused")
Peter van 't Hof's avatar
Peter van 't Hof committed
245
    writeMapToJsonFile(_defaults, "defaults")
246
    writeMapToJsonFile(found, "found")
247
    writeMapToJsonFile(fixed, "fixed")
248
    writeMapToJsonFile(effectiveFound, "effective.found")
249
    writeMapToJsonFile(effectiveFixed, "effective.fixed")
250 251 252 253 254 255
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

256
  override def toString: String = map.toString()
Peter van 't Hof's avatar
Peter van 't Hof committed
257 258
}

Peter van 't Hof's avatar
Peter van 't Hof committed
259
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
260 261
  val global = new Config

262 263 264
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
265
   * @param config2 Low prio map
266 267
   * @return Merged config
   */
268
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
Peter van 't Hof's avatar
Peter van 't Hof committed
269

270 271 272
  /**
   * Search for value in index position in a map
   * @param map Map to search in
273
   * @param startIndex Config index
274 275
   * @return Value
   */
276 277 278 279 280 281 282 283 284 285 286 287 288
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
289
      if (p.isDefined) p
290 291 292 293 294
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
295 296
      }
    }
297 298 299 300 301 302 303 304 305 306

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
307
    }
308 309 310 311 312 313 314 315

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

316
    tailSearch(startIndex.path)
317
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
318
}