Config.scala 11.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
15
package nl.lumc.sasc.biopet.utils.config
Peter van 't Hof's avatar
Peter van 't Hof committed
16

Peter van 't Hof's avatar
Peter van 't Hof committed
17
import java.io.{ File, PrintWriter }
18
19

import nl.lumc.sasc.biopet.utils.{ConfigUtils, Logging}
20
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
21

22
23
24
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

25
26
/**
 * This class can store nested config values
27
 * @param _map Map with value for new config
28
29
 * @constructor Load config with existing map
 */
30
class Config(protected var _map: Map[String, Any],
Peter van 't Hof's avatar
Peter van 't Hof committed
31
             protected var _defaults: Map[String, Any] = Map()) extends Logging {
32
  logger.debug("Init phase of config")
33

Peter van 't Hof's avatar
Peter van 't Hof committed
34
  /** Default constructor */
Peter van 't Hof's avatar
Peter van 't Hof committed
35
36
37
38
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
39

40
  def map = _map
Peter van 't Hof's avatar
Peter van 't Hof committed
41
  def defaults = _defaults
42

43
44
45
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
Peter van 't Hof's avatar
Peter van 't Hof committed
46
   * @param default if true files are added to default instead of normal map
47
   */
48
  def loadConfigEnv(valueName: String, default: Boolean) {
Peter van 't Hof's avatar
Peter van 't Hof committed
49
    sys.env.get(valueName) match {
50
      case Some(globalFiles) =>
Peter van 't Hof's avatar
Peter van 't Hof committed
51
52
        for (globalFile <- globalFiles.split(":")) {
          val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
53
          if (file.exists) {
54
            logger.debug("Loading config file: " + file)
55
            loadConfigFile(file, default)
Peter van 't Hof's avatar
Peter van 't Hof committed
56
57
          } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
        }
58
      case _ => logger.debug(valueName + " value not found, no global config is loaded")
59
60
61
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
62
  /** Loading default value for biopet */
63
  def loadDefaultConfig() {
64
    loadConfigEnv("BIOPET_CONFIG", default = true)
65
66
  }

67
68
69
70
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
71
  def loadConfigFile(configFile: File, default: Boolean = false) {
72
    val configMap = fileToConfigMap(configFile)
73
    if (default) {
Peter van 't Hof's avatar
Peter van 't Hof committed
74
75
76
      if (_defaults.isEmpty) _defaults = configMap
      else _defaults = mergeMaps(configMap, _defaults)
      logger.debug("New defaults: " + _defaults)
77
    } else {
78
79
80
      if (_map.isEmpty) _map = configMap
      else _map = mergeMaps(configMap, _map)
      logger.debug("New config: " + _map)
81
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
82
  }
bow's avatar
bow committed
83

84
85
86
87
88
89
90
91
92
  /**
   * Add a single vallue to the config
   * @param key key of value
   * @param value value itself
   * @param path Path to value
   * @param default if true value is put in default map
   */
  def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
    val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
Peter van 't Hof's avatar
Peter van 't Hof committed
93
    if (default) _defaults = mergeMaps(valueMap, _defaults)
94
    else _map = mergeMaps(valueMap, _map)
95
96
  }

97
  protected[config] var notFoundCache: Set[ConfigValueIndex] = Set()
98
  protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
99
100
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
101
  protected[config] def clearCache(): Unit = {
102
    notFoundCache = Set()
103
104
105
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
106

107
108
109
110
111
112
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
113
  def contains(s: String): Boolean = _map.contains(s)
114
115
116
117
118
119

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
120
121
122
123
124
125
126
127
128
  def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @param fixedValues Fixed values
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
129
    if (notFoundCache.contains(requestedIndex)) false
130
    else if (fixedCache.contains(requestedIndex)) true
131
    else if (foundCache.contains(requestedIndex)) true
Peter van 't Hof's avatar
Peter van 't Hof committed
132
    else {
133
134
135
      val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
      if (fixedValue.isDefined) {
        fixedCache += (requestedIndex -> fixedValue.get)
136
        true
137
      } else {
138
        val value = Config.getValueFromMap(_map, requestedIndex)
139
140
141
142
        if (value.isDefined && value.get.value != None) {
          foundCache += (requestedIndex -> value.get)
          true
        } else {
143
          notFoundCache += requestedIndex
144
145
          false
        }
Peter van 't Hof's avatar
Peter van 't Hof committed
146
147
      }
    }
148
149
150
151
152
153
154
155
156

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
157
158
159
160
  def contains(module: String, path: List[String],
               key: String,
               freeVar: Boolean = true,
               fixedValues: Map[String, Any] = Map()): Boolean = {
161
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
162
    contains(requestedIndex, fixedValues)
Peter van 't Hof's avatar
Peter van 't Hof committed
163
  }
bow's avatar
bow committed
164

165
166
167
168
169
170
171
172
173
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
174
175
176
177
178
179
  def apply(module: String,
            path: List[String],
            key: String,
            default: Any = null,
            freeVar: Boolean = true,
            fixedValues: Map[String, Any] = Map()): ConfigValue = {
180
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
181
182
183
184
185
186
187
188
189
    if (contains(requestedIndex, fixedValues)) {
      val fixedValue = fixedCache.get(requestedIndex)
      if (fixedValue.isDefined) {
        val userValue = Config.getValueFromMap(_map, requestedIndex)
        if (userValue.isDefined)
          logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
      }

      fixedValue.getOrElse(foundCache(requestedIndex))
Peter van 't Hof's avatar
Peter van 't Hof committed
190
    } else if (default != null) {
191
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
192
193
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
194
  }
bow's avatar
bow committed
195

196
  def writeReport(directory: File): Future[Unit] = Future {
Peter van 't Hof's avatar
Peter van 't Hof committed
197
    directory.mkdirs()
198
199
200
201
202
203
204
205
206
207
208
209
210
211

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
212
      val file = new File(directory, name + ".json")
213
214
215
216
217
218
219
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
220
    val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
Peter van 't Hof's avatar
Peter van 't Hof committed
221
    val unused = uniqueKeys(map, found)
222
223
224
225
226
227
228
229
230
231

    def reportUnused(map: Map[String, Any], path: List[String] = Nil): Unit = {
      map.foreach {
        case (key, value: Map[_, _]) => reportUnused(value.asInstanceOf[Map[String, Any]], path :+ key)
        case (key, value) => logger.warn(s"config key in user config is never used, key: $key" +
          (if (path.nonEmpty) s", path: ${path.mkString(" -> ")}" else ""))
      }
    }

    reportUnused(unused)
232
233
234

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
235
    val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
236
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
237
    val notFound = convertIndexValuesToMap(notFoundCache.toList.map((_, None)), Some(false))
238
239
240
241
242

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

243
244
    writeMapToJsonFile(map, "input")
    writeMapToJsonFile(unused, "unused")
Peter van 't Hof's avatar
Peter van 't Hof committed
245
    writeMapToJsonFile(_defaults, "defaults")
246
    writeMapToJsonFile(found, "found")
247
    writeMapToJsonFile(fixed, "fixed")
248
    writeMapToJsonFile(effectiveFound, "effective.found")
249
    writeMapToJsonFile(effectiveFixed, "effective.fixed")
250
251
252
253
254
255
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

256
  override def toString: String = map.toString()
Peter van 't Hof's avatar
Peter van 't Hof committed
257
258
}

Peter van 't Hof's avatar
Peter van 't Hof committed
259
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
260
261
  val global = new Config

262
263
264
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
265
   * @param config2 Low prio map
266
267
   * @return Merged config
   */
268
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
Peter van 't Hof's avatar
Peter van 't Hof committed
269

270
271
272
  /**
   * Search for value in index position in a map
   * @param map Map to search in
273
   * @param startIndex Config index
274
275
   * @return Value
   */
276
277
278
279
280
281
282
283
284
285
286
287
288
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
289
      if (p.isDefined) p
290
291
292
293
294
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
295
296
      }
    }
297
298
299
300
301
302
303
304
305
306

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
307
    }
308
309
310
311
312
313
314
315

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

316
    tailSearch(startIndex.path)
317
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
318
}