Config.scala 9.04 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
16
17
package nl.lumc.sasc.biopet.core.config

18
import java.io.{ PrintWriter, File }
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.core.Logging
20
import nl.lumc.sasc.biopet.utils.ConfigUtils
21
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
22

23
24
import scala.reflect.io.Directory

25
26
27
28
29
/**
 * This class can store nested config values
 * @param map Map with value for new config
 * @constructor Load config with existing map
 */
30
31
class Config(var map: Map[String, Any],
             protected[core] var defaults: Map[String, Any] = Map()) extends Logging {
32
  logger.debug("Init phase of config")
33
34
35
36

  /**
   * Default constructor
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
37
38
39
40
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
41

42
43
44
45
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
   */
46
  def loadConfigEnv(valueName: String, default: Boolean) {
Peter van 't Hof's avatar
Peter van 't Hof committed
47
48
49
50
    sys.env.get(valueName) match {
      case Some(globalFiles) => {
        for (globalFile <- globalFiles.split(":")) {
          val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
51
          if (file.exists) {
Peter van 't Hof's avatar
Peter van 't Hof committed
52
            logger.info("Loading config file: " + file)
53
            loadConfigFile(file, default)
Peter van 't Hof's avatar
Peter van 't Hof committed
54
55
56
57
          } else logger.warn(valueName + " value found but file '" + file + "' does not exist, no global config is loaded")
        }
      }
      case _ => logger.info(valueName + " value not found, no global config is loaded")
58
59
60
    }
  }

61
62
63
  /**
   * Loading default value for biopet
   */
64
  def loadDefaultConfig() {
65
    loadConfigEnv("BIOPET_CONFIG", true)
66
67
  }

68
69
70
71
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
72
  def loadConfigFile(configFile: File, default: Boolean = false) {
73
    val configMap = fileToConfigMap(configFile)
74
75
76
77
78
79
80
81
82
    if (default) {
      if (defaults.isEmpty) defaults = configMap
      else defaults = mergeMaps(configMap, defaults)
      logger.debug("New defaults: " + defaults)
    } else {
      if (map.isEmpty) map = configMap
      else map = mergeMaps(configMap, map)
      logger.debug("New config: " + map)
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
83
  }
bow's avatar
bow committed
84

85
86
87
  protected[config] var notFoundCache: List[ConfigValueIndex] = List()
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
88
89
90
91
92
  protected[config] def clearCache: Unit = {
    notFoundCache = List()
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
93

94
95
96
97
98
99
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
bow's avatar
bow committed
100
  def contains(s: String): Boolean = map.contains(s)
101
102
103
104
105
106
107

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex): Boolean =
Peter van 't Hof's avatar
Peter van 't Hof committed
108
109
110
    if (notFoundCache.contains(requestedIndex)) return false
    else if (foundCache.contains(requestedIndex)) return true
    else {
111
      val value = Config.getValueFromMap(map, requestedIndex)
112
      if (value.isDefined && value.get.value != None) {
113
        foundCache += (requestedIndex -> value.get)
Peter van 't Hof's avatar
Peter van 't Hof committed
114
        return true
115
      } else {
Peter van 't Hof's avatar
Peter van 't Hof committed
116
117
118
119
        notFoundCache +:= requestedIndex
        return false
      }
    }
120
121
122
123
124
125
126
127
128
129
130
131

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
  def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
    contains(requestedIndex)
Peter van 't Hof's avatar
Peter van 't Hof committed
132
  }
bow's avatar
bow committed
133

134
135
136
137
138
139
140
141
142
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
143
  protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
144
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
145
    if (contains(requestedIndex)) foundCache(requestedIndex)
146
    else if (default != null) {
147
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
148
149
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
150
  }
bow's avatar
bow committed
151

Peter van 't Hof's avatar
Peter van 't Hof committed
152
153
  def writeReport(id: String, directory: File): Unit = {
    directory.mkdirs()
154
155
156
157
158
159
160
161
162
163
164
165
166
167

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
168
      val file = new File(directory, id + "." + name + ".json")
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
    val notFound = convertIndexValuesToMap(notFoundCache.map((_, None)), Some(false))

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

186
    writeMapToJsonFile(this.map, "input")
187
188
189
190
191
192
193
194
    writeMapToJsonFile(found, "found")
    writeMapToJsonFile(effectiveFound, "effective.found")
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

bow's avatar
bow committed
195
  override def toString(): String = map.toString
Peter van 't Hof's avatar
Peter van 't Hof committed
196
197
}

Peter van 't Hof's avatar
Peter van 't Hof committed
198
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
199
200
  val global = new Config

201
202
203
204
205
206
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
   * @param config2
   * @return Merged config
   */
207
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
Peter van 't Hof's avatar
Peter van 't Hof committed
208

209
210
211
  /**
   * Search for value in index position in a map
   * @param map Map to search in
212
   * @param startIndex Config index
213
214
   * @return Value
   */
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p != None) p
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
234
235
      }
    }
236
237
238
239
240
241
242
243
244
245

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
246
    }
247
248
249
250
251
252
253
254
255

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

    return tailSearch(startIndex.path)
256
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
257
}