Config.scala 11.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
16
package nl.lumc.sasc.biopet.utils.config
Peter van 't Hof's avatar
Peter van 't Hof committed
17

Peter van 't Hof's avatar
Peter van 't Hof committed
18
import java.io.{ File, PrintWriter }
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.utils.{ Logging, ConfigUtils }
20
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
21

22
23
/**
 * This class can store nested config values
24
 * @param _map Map with value for new config
25
26
 * @constructor Load config with existing map
 */
27
class Config(protected var _map: Map[String, Any],
Peter van 't Hof's avatar
Peter van 't Hof committed
28
             protected var _defaults: Map[String, Any] = Map()) extends Logging {
29
  logger.debug("Init phase of config")
30

Peter van 't Hof's avatar
Peter van 't Hof committed
31
  /** Default constructor */
Peter van 't Hof's avatar
Peter van 't Hof committed
32
33
34
35
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
36

37
  def map = _map
Peter van 't Hof's avatar
Peter van 't Hof committed
38
  def defaults = _defaults
39

40
41
42
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
Peter van 't Hof's avatar
Peter van 't Hof committed
43
   * @param default if true files are added to default instead of normal map
44
   */
45
  def loadConfigEnv(valueName: String, default: Boolean) {
Peter van 't Hof's avatar
Peter van 't Hof committed
46
    sys.env.get(valueName) match {
47
      case Some(globalFiles) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
48
49
        for (globalFile <- globalFiles.split(":")) {
          val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
50
          if (file.exists) {
Peter van 't Hof's avatar
Peter van 't Hof committed
51
            logger.info("Loading config file: " + file)
52
            loadConfigFile(file, default)
Peter van 't Hof's avatar
Peter van 't Hof committed
53
54
55
          } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
        }
      case _ => logger.info(valueName + " value not found, no global config is loaded")
56
57
58
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
59
  /** Loading default value for biopet */
60
  def loadDefaultConfig() {
61
    loadConfigEnv("BIOPET_CONFIG", default = true)
62
63
  }

64
65
66
67
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
68
  def loadConfigFile(configFile: File, default: Boolean = false) {
69
    val configMap = fileToConfigMap(configFile)
70
    if (default) {
Peter van 't Hof's avatar
Peter van 't Hof committed
71
72
73
      if (_defaults.isEmpty) _defaults = configMap
      else _defaults = mergeMaps(configMap, _defaults)
      logger.debug("New defaults: " + _defaults)
74
    } else {
75
76
77
      if (_map.isEmpty) _map = configMap
      else _map = mergeMaps(configMap, _map)
      logger.debug("New config: " + _map)
78
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
79
  }
bow's avatar
bow committed
80

81
82
83
84
85
86
87
88
89
  /**
   * Add a single vallue to the config
   * @param key key of value
   * @param value value itself
   * @param path Path to value
   * @param default if true value is put in default map
   */
  def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
    val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
Peter van 't Hof's avatar
Peter van 't Hof committed
90
    if (default) _defaults = mergeMaps(valueMap, _defaults)
91
    else _map = mergeMaps(valueMap, _map)
92
93
  }

94
  protected[config] var notFoundCache: Set[ConfigValueIndex] = Set()
95
  protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
96
97
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
98
  protected[config] def clearCache(): Unit = {
99
    notFoundCache = Set()
100
101
102
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
103

104
105
106
107
108
109
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
110
  def contains(s: String): Boolean = _map.contains(s)
111
112
113
114
115
116

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
117
118
119
120
121
122
123
124
125
  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @param fixedValues Fixed values
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
126
    if (notFoundCache.contains(requestedIndex)) false
127
    else if (fixedCache.contains(requestedIndex)) true
128
    else if (foundCache.contains(requestedIndex)) true
Peter van 't Hof's avatar
Peter van 't Hof committed
129
    else {
130
131
132
      val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
      if (fixedValue.isDefined) {
        fixedCache += (requestedIndex -> fixedValue.get)
133
        true
134
      } else {
135
        val value = Config.getValueFromMap(_map, requestedIndex)
136
137
138
139
        if (value.isDefined && value.get.value != None) {
          foundCache += (requestedIndex -> value.get)
          true
        } else {
140
          notFoundCache += requestedIndex
141
142
          false
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
143
144
      }
    }
145
146
147
148
149
150
151
152
153

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
154
155
156
157
  def contains(module: String, path: List[String],
               key: String,
               freeVar: Boolean = true,
               fixedValues: Map[String, Any] = Map()): Boolean = {
158
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
159
    contains(requestedIndex, fixedValues)
Peter van 't Hof's avatar
Peter van 't Hof committed
160
  }
bow's avatar
bow committed
161

162
163
164
165
166
167
168
169
170
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
171
172
173
174
175
176
  def apply(module: String,
            path: List[String],
            key: String,
            default: Any = null,
            freeVar: Boolean = true,
            fixedValues: Map[String, Any] = Map()): ConfigValue = {
177
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
178
179
180
181
182
183
184
185
186
    if (contains(requestedIndex, fixedValues)) {
      val fixedValue = fixedCache.get(requestedIndex)
      if (fixedValue.isDefined) {
        val userValue = Config.getValueFromMap(_map, requestedIndex)
        if (userValue.isDefined)
          logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
      }

      fixedValue.getOrElse(foundCache(requestedIndex))
Peter van 't Hof's avatar
Peter van 't Hof committed
187
    } else if (default != null) {
188
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
189
190
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
191
  }
bow's avatar
bow committed
192

Peter van 't Hof's avatar
Peter van 't Hof committed
193
194
  def writeReport(id: String, directory: File): Unit = {
    directory.mkdirs()
195
196
197
198
199
200
201
202
203
204
205
206
207
208

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
209
      val file = new File(directory, id + "." + name + ".json")
210
211
212
213
214
215
216
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
217
    val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
Peter van 't Hof's avatar
Peter van 't Hof committed
218
    val unused = uniqueKeys(map, found)
219
220
221
222
223
224
225
226
227
228

    def reportUnused(map: Map[String, Any], path: List[String] = Nil): Unit = {
      map.foreach {
        case (key, value: Map[_, _]) => reportUnused(value.asInstanceOf[Map[String, Any]], path :+ key)
        case (key, value) => logger.warn(s"config key in user config is never used, key: $key" +
          (if (path.nonEmpty) s", path: ${path.mkString(" -> ")}" else ""))
      }
    }

    reportUnused(unused)
229
230
231

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
232
    val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
233
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
234
    val notFound = convertIndexValuesToMap(notFoundCache.toList.map((_, None)), Some(false))
235
236
237
238
239

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

240
241
    writeMapToJsonFile(map, "input")
    writeMapToJsonFile(unused, "unused")
Peter van 't Hof's avatar
Peter van 't Hof committed
242
    writeMapToJsonFile(_defaults, "defaults")
243
    writeMapToJsonFile(found, "found")
244
    writeMapToJsonFile(fixed, "fixed")
245
    writeMapToJsonFile(effectiveFound, "effective.found")
246
    writeMapToJsonFile(effectiveFixed, "effective.fixed")
247
248
249
250
251
252
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

253
  override def toString: String = map.toString()
Peter van 't Hof's avatar
Peter van 't Hof committed
254
255
}

Peter van 't Hof's avatar
Peter van 't Hof committed
256
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
257
258
  val global = new Config

259
260
261
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
262
   * @param config2 Low prio map
263
264
   * @return Merged config
   */
265
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
Peter van 't Hof's avatar
Peter van 't Hof committed
266

267
268
269
  /**
   * Search for value in index position in a map
   * @param map Map to search in
270
   * @param startIndex Config index
271
272
   * @return Value
   */
273
274
275
276
277
278
279
280
281
282
283
284
285
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
286
      if (p.isDefined) p
287
288
289
290
291
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
292
293
      }
    }
294
295
296
297
298
299
300
301
302
303

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
304
    }
305
306
307
308
309
310
311
312

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

313
    tailSearch(startIndex.path)
314
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
315
}