Config.scala 8.67 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
16
17
package nl.lumc.sasc.biopet.core.config

18
import java.io.{ PrintWriter, File }
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import nl.lumc.sasc.biopet.core.Logging
20
import nl.lumc.sasc.biopet.utils.ConfigUtils
21
import nl.lumc.sasc.biopet.utils.ConfigUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
22

23
24
import scala.reflect.io.Directory

25
26
27
28
29
/**
 * This class can store nested config values
 * @param map Map with value for new config
 * @constructor Load config with existing map
 */
bow's avatar
bow committed
30
class Config(var map: Map[String, Any]) extends Logging {
31
  logger.debug("Init phase of config")
32
33
34
35

  /**
   * Default constructor
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
36
37
38
39
  def this() = {
    this(Map())
    loadDefaultConfig()
  }
bow's avatar
bow committed
40

41
42
43
44
  /**
   * Loading a environmental variable as location of config files to merge into the config
   * @param valueName Name of value
   */
45
  def loadConfigEnv(valueName: String) {
Peter van 't Hof's avatar
Peter van 't Hof committed
46
47
    val globalFiles = sys.env.get(valueName).getOrElse("").split(":")
    if (globalFiles.isEmpty) logger.info(valueName + " value not found, no global config is loaded")
48
    for (globalFile <- globalFiles) {
49
      val file: File = new File(globalFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
50
51
52
      if (file.exists()) {
        logger.info("Loading config file: " + file)
        loadConfigFile(file)
53
54
55
56
      } else logger.warn(valueName + " value found but file does not exist, no global config is loaded")
    }
  }

57
58
59
  /**
   * Loading default value for biopet
   */
60
61
62
63
  def loadDefaultConfig() {
    loadConfigEnv("BIOPET_CONFIG")
  }

64
65
66
67
  /**
   * Merge a json file into the config
   * @param configFile Location of file
   */
bow's avatar
bow committed
68
  def loadConfigFile(configFile: File) {
69
    val configMap = fileToConfigMap(configFile)
bow's avatar
bow committed
70

71
72
    if (map.isEmpty) map = configMap
    else map = mergeMaps(configMap, map)
Peter van 't Hof's avatar
Peter van 't Hof committed
73
74
    logger.debug("New config: " + map)
  }
bow's avatar
bow committed
75

76
77
78
  protected[config] var notFoundCache: List[ConfigValueIndex] = List()
  protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
  protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
79
80
81
82
83
  protected[config] def clearCache: Unit = {
    notFoundCache = List()
    foundCache = Map()
    defaultCache = Map()
  }
bow's avatar
bow committed
84

85
86
87
88
89
90
  /**
   * Check if value exist in root of config
   * @deprecated
   * @param s key
   * @return True if exist
   */
bow's avatar
bow committed
91
  def contains(s: String): Boolean = map.contains(s)
92
93
94
95
96
97
98

  /**
   * Checks if value exist in config
   * @param requestedIndex Index to value
   * @return True if exist
   */
  def contains(requestedIndex: ConfigValueIndex): Boolean =
Peter van 't Hof's avatar
Peter van 't Hof committed
99
100
101
    if (notFoundCache.contains(requestedIndex)) return false
    else if (foundCache.contains(requestedIndex)) return true
    else {
102
      val value = Config.getValueFromMap(map, requestedIndex)
103
      if (value.isDefined && value.get.value != None) {
104
        foundCache += (requestedIndex -> value.get)
Peter van 't Hof's avatar
Peter van 't Hof committed
105
        return true
106
      } else {
Peter van 't Hof's avatar
Peter van 't Hof committed
107
108
109
110
        notFoundCache +:= requestedIndex
        return false
      }
    }
111
112
113
114
115
116
117
118
119
120
121
122

  /**
   * Checks if value exist in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param freeVar Default true, if set false value must exist in module
   * @return True if exist
   */
  def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
    contains(requestedIndex)
Peter van 't Hof's avatar
Peter van 't Hof committed
123
  }
bow's avatar
bow committed
124

125
126
127
128
129
130
131
132
133
  /**
   * Find value in config
   * @param module Name of module
   * @param path Path to start searching
   * @param key Name of value
   * @param default Default value when no value is found
   * @param freeVar Default true, if set false value must exist in module
   * @return Config value
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
134
  protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
135
    val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
136
    if (contains(requestedIndex)) foundCache(requestedIndex)
137
    else if (default != null) {
138
      defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
139
140
      defaultCache(requestedIndex)
    } else ConfigValue(requestedIndex, null, null, freeVar)
Peter van 't Hof's avatar
Peter van 't Hof committed
141
  }
bow's avatar
bow committed
142

Peter van 't Hof's avatar
Peter van 't Hof committed
143
144
  def writeReport(id: String, directory: File): Unit = {
    directory.mkdirs()
145
146
147
148
149
150
151
152
153
154
155
156
157
158

    def convertIndexValuesToMap(input: List[(ConfigValueIndex, Any)], forceFreeVar: Option[Boolean] = None): Map[String, Any] = {
      input.foldLeft(Map[String, Any]())(
        (a: Map[String, Any], x: (ConfigValueIndex, Any)) => {
          val v = {
            if (forceFreeVar.getOrElse(x._1.freeVar)) Map(x._1.key -> x._2)
            else Map(x._1.module -> Map(x._1.key -> x._2))
          }
          val newMap = x._1.path.foldRight(v)((p, map) => Map(p -> map))
          ConfigUtils.mergeMaps(a, newMap)
        })
    }

    def writeMapToJsonFile(map: Map[String, Any], name: String): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
159
      val file = new File(directory, id + "." + name + ".json")
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
      val writer = new PrintWriter(file)
      writer.write(ConfigUtils.mapToJson(map).spaces2)
      writer.close()
    }

    // Positions where values are found
    val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))

    // Positions where to start searching
    val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
    val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
    val notFound = convertIndexValuesToMap(notFoundCache.map((_, None)), Some(false))

    // Merged maps
    val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
    val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)

177
    writeMapToJsonFile(this.map, "input")
178
179
180
181
182
183
184
185
    writeMapToJsonFile(found, "found")
    writeMapToJsonFile(effectiveFound, "effective.found")
    writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
    writeMapToJsonFile(notFound, "not.found")
    writeMapToJsonFile(fullEffective, "effective.full")
    writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
  }

bow's avatar
bow committed
186
  override def toString(): String = map.toString
Peter van 't Hof's avatar
Peter van 't Hof committed
187
188
}

Peter van 't Hof's avatar
Peter van 't Hof committed
189
object Config extends Logging {
Peter van 't Hof's avatar
Peter van 't Hof committed
190
191
  val global = new Config

192
193
194
195
196
197
  /**
   * Merge 2 config objects
   * @param config1 prio over config 2
   * @param config2
   * @return Merged config
   */
198
  def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
Peter van 't Hof's avatar
Peter van 't Hof committed
199

200
201
202
  /**
   * Search for value in index position in a map
   * @param map Map to search in
203
   * @param startIndex Config index
204
205
   * @return Value
   */
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
  def getValueFromMap(map: Map[String, Any], startIndex: ConfigValueIndex): Option[ConfigValue] = {
    def getFromPath(path: List[String]): Option[ConfigValue] = {
      val p = getValueFromPath(map, path ::: startIndex.module :: startIndex.key :: Nil)
      if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = false), p.get))
      else if (startIndex.freeVar) {
        val p = getValueFromPath(map, path ::: startIndex.key :: Nil)
        if (p.isDefined) Option(ConfigValue(startIndex, ConfigValueIndex(startIndex.module, path, startIndex.key, freeVar = true), p.get))
        else None
      } else None
    }

    def tailSearch(path: List[String]): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p != None) p
      else if (path == Nil) None
      else {
        val p = initSearch(path)
        if (p.isDefined) p
        else tailSearch(path.tail)
225
226
      }
    }
227
228
229
230
231
232
233
234
235
236

    def initSearch(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path)
      if (p.isDefined) p
      else if (path == Nil) None
      else {
        val p = skipNested(path, tail)
        if (p.isDefined) p
        else initSearch(path.init, path.last :: tail)
      }
237
    }
238
239
240
241
242
243
244
245
246

    def skipNested(path: List[String], tail: List[String] = Nil): Option[ConfigValue] = {
      val p = getFromPath(path ::: tail)
      if (p.isDefined) p
      else if (tail == Nil) None
      else skipNested(path, tail.tail)
    }

    return tailSearch(startIndex.path)
247
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
248
}