ConfigUtils.scala 14.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
18
package nl.lumc.sasc.biopet.utils

import java.io.File
Peter van 't Hof's avatar
Peter van 't Hof committed
19
20
21
22

import argonaut.Argonaut._
import argonaut._
import nl.lumc.sasc.biopet.core.{ BiopetQScript, Logging }
23
import nl.lumc.sasc.biopet.core.config.ConfigValue
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.yaml.snakeyaml.Yaml
Peter van 't Hof's avatar
Peter van 't Hof committed
25

Peter van 't Hof's avatar
Peter van 't Hof committed
26
import scala.collection.JavaConversions._
27

28
29
30
31
/**
 * This object contains general function for the config
 *
 */
32
object ConfigUtils extends Logging {
33
34
35
36
37
38
  /**
   * Merge 2 maps, when value is in a map in map1 and map2 the value calls recursively this function
   * @param map1 Prio over map2
   * @param map2 Backup for map1
   * @return merged map
   */
39
40
  def mergeMaps(map1: Map[String, Any], map2: Map[String, Any],
                resolveConflict: (Any, Any, String) => Any = (m1, m2, key) => m1): Map[String, Any] = {
41
42
43
44
45
46
    var newMap: Map[String, Any] = Map()
    for (key <- map1.keySet.++(map2.keySet)) {
      if (!map2.contains(key)) newMap += (key -> map1(key))
      else if (!map1.contains(key)) newMap += (key -> map2(key))
      else {
        map1(key) match {
Peter van 't Hof's avatar
Peter van 't Hof committed
47
          case m1: Map[_, _] =>
48
            map2(key) match {
Peter van 't Hof's avatar
Peter van 't Hof committed
49
              case m2: Map[_, _] => newMap += (key -> mergeMaps(any2map(m1), any2map(m2), resolveConflict))
50
51
              case _             => newMap += (key -> map1(key))
            }
52
          case _ => newMap += (key -> resolveConflict(map1(key), map2(key), key))
53
54
55
        }
      }
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
56
    newMap
57
58
  }

59
60
61
62
63
64
  /**
   * Get nested map
   * @param map Map to search in
   * @param path Nested path to get from map
   * @return Nested map
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
65
66
  def getMapFromPath(map: Map[String, Any], path: List[String]): Option[Map[String, Any]] = {
    val value = getValueFromPath(map, path) getOrElse { return None }
67
    value match {
Peter van 't Hof's avatar
Peter van 't Hof committed
68
      case m: Map[_, _] => Some(m.asInstanceOf[Map[String, Any]])
69
      case _            => throw new IllegalStateException("Value is not a map: " + value)
70
    }
71
72
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
73
74
75
76
77
78
  /**
   * Get nested value
   * @param map Map to search in
   * @param path Path to the value
   * @return Some(value) or None if not found
   */
79
  def getValueFromPath(map: Map[String, Any], path: List[String]): Option[Any] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
80
    val value = map.get(path.head)
Peter van 't Hof's avatar
Peter van 't Hof committed
81
    if (path.tail.isEmpty || value.isEmpty) value
Peter van 't Hof's avatar
Peter van 't Hof committed
82
    else value.get match {
Peter van 't Hof's avatar
Peter van 't Hof committed
83
84
85
      case map: Map[_, _]                     => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
      case map: java.util.LinkedHashMap[_, _] => getValueFromPath(map.toMap.asInstanceOf[Map[String, Any]], path.tail)
      case _                                  => None
Peter van 't Hof's avatar
Peter van 't Hof committed
86
    }
87
88
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
89
  /** Make json aboject from a file */
90
  def fileToJson(configFile: File): Json = {
91
92
93
    logger.debug("Jsonfile: " + configFile)
    val jsonText = scala.io.Source.fromFile(configFile).mkString
    val json = Parse.parseOption(jsonText)
94
95
96
    logger.debug(json)

    json getOrElse {
97
98
      throw new IllegalStateException("The config JSON file is either not properly formatted or not a JSON file, file: " + configFile)
    }
99
100
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
101
  /** Convert config value to map */
102
  def fileToConfigMap(configFile: File): Map[String, Any] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
103
104
105
106
107
108

    val configMap = {
      if (configFile.getName.endsWith(".yaml")) yamlToMap(configFile)
      else jsonToMap(fileToJson(configFile))
    }
    logger.debug("Contain: " + configMap)
Peter van 't Hof's avatar
Peter van 't Hof committed
109
    configMap
Peter van 't Hof's avatar
Peter van 't Hof committed
110
111
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
112
  /** Convert a yaml file to map[String, Any] */
Peter van 't Hof's avatar
Peter van 't Hof committed
113
114
115
116
  def yamlToMap(file: File): Map[String, Any] = {
    val yaml = new Yaml()
    val a = yaml.load(scala.io.Source.fromFile(file).reader())
    ConfigUtils.any2map(a)
117
118
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
119
  /** Convert json to native scala map/values */
120
121
122
123
124
125
126
  def jsonToMap(json: Json): Map[String, Any] = {
    var output: Map[String, Any] = Map()
    if (json.isObject) {
      for (key <- json.objectFieldsOrEmpty) {
        val value: Any = jsonToAny(json.field(key).get)
        output += (key -> value)
      }
127
    } else throw new IllegalStateException("Given value is no json object: " + json)
Peter van 't Hof's avatar
Peter van 't Hof committed
128
    output
129
130
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
131
  /** Convert json value to native scala value */
132
  def jsonToAny(json: Json): Any = {
Peter van 't Hof's avatar
Peter van 't Hof committed
133
    if (json.isObject) jsonToMap(json)
134
135
136
    else if (json.isArray) {
      var list: List[Any] = List()
      for (value <- json.array.get) list ::= jsonToAny(value)
Peter van 't Hof's avatar
Peter van 't Hof committed
137
138
139
      list.reverse
    } else if (json.isBool) json.bool.get
    else if (json.isString) json.string.get.toString
140
141
    else if (json.isNumber) {
      val num = json.number.get
Peter van 't Hof's avatar
Peter van 't Hof committed
142
143
144
      if (num % 1 > 0) num.toDouble
      else num.toLong
    } else if (json.isNull) None
145
    else throw new IllegalStateException("Config value type not supported, value: " + json)
146
147
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
148
  /** Convert native scala map to json */
149
150
151
152
153
154
155
156
157
  def mapToJson(map: Map[String, Any]): Json = {
    map.foldLeft(jEmptyObject)((acc, kv) => (kv._1 := {
      kv._2 match {
        case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
        case _            => anyToJson(kv._2)
      }
    }) ->: acc)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
158
  /** Convert native scala value to json, fall back on .toString if type is not a native scala value */
159
160
161
  def anyToJson(any: Any): Json = {
    any match {
      case j: Json      => j
Peter van 't Hof's avatar
Peter van 't Hof committed
162
      case None         => Json.jNull
163
      case Some(x)      => anyToJson(x)
164
      case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
Peter van 't Hof's avatar
Peter van 't Hof committed
165
      case l: List[_]   => Json.array(l.map(anyToJson): _*)
166
      case b: Boolean   => Json.jBool(b)
167
168
169
170
171
172
      case n: Int       => Json.jNumberOrString(n)
      case n: Double    => Json.jNumberOrString(n)
      case n: Long      => Json.jNumberOrString(n)
      case n: Short     => Json.jNumberOrString(n)
      case n: Float     => Json.jNumberOrString(n)
      case n: Byte      => Json.jNumberOrString(n)
Sander van der Zeeuw's avatar
Sander van der Zeeuw committed
173
      case null         => Json.jNull
174
175
176
177
      case _            => jString(any.toString)
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
178
  /** Convert Any to String */
179
180
181
  def any2string(any: Any): String = {
    if (any == null) return null
    any match {
182
183
      case s: String => s
      case _         => any.toString
184
185
186
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
187
  /** Convert Any to Int */
188
189
  def any2int(any: Any): Int = {
    any match {
190
      case i: Int    => i
191
      case i: Double => i.toInt
192
      case i: Long   => i.toInt
Peter van 't Hof's avatar
Peter van 't Hof committed
193
      case i: String =>
194
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
195
        i.toInt
196
197
198
199
      case _ => throw new IllegalStateException("Value '" + any + "' is not an int")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
200
  /** Convert Any to Long */
201
202
  def any2long(any: Any): Long = {
    any match {
203
204
205
      case l: Double => l.toLong
      case l: Int    => l.toLong
      case l: Long   => l
Peter van 't Hof's avatar
Peter van 't Hof committed
206
      case l: String =>
207
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
208
        l.toLong
209
210
211
212
      case _ => throw new IllegalStateException("Value '" + any + "' is not an int")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
213
  /** Convert Any to Double */
214
215
  def any2double(any: Any): Double = {
    any match {
216
217
218
      case d: Double => d
      case d: Float  => d.toDouble
      case d: Int    => d.toDouble
219
      case f: Long   => f.toDouble
Peter van 't Hof's avatar
Peter van 't Hof committed
220
      case d: String =>
221
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
Peter van 't Hof's avatar
Peter van 't Hof committed
222
        d.toDouble
223
      case _ => throw new IllegalStateException("Value '" + any + "' is not an number")
224
225
226
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
227
  /** Convert Any to Float */
228
229
  def any2float(any: Any): Float = {
    any match {
230
231
      case f: Double => f.toFloat
      case f: Int    => f.toFloat
232
      case f: Long   => f.toFloat
233
      case f: Float  => f
Peter van 't Hof's avatar
Peter van 't Hof committed
234
      case f: String =>
235
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
236
        f.toFloat
237
      case _ => throw new IllegalStateException("Value '" + any + "' is not an number")
238
239
240
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
241
  /** Convert Any to Boolean */
242
243
  def any2boolean(any: Any): Boolean = {
    any match {
244
      case b: Boolean => b
Peter van 't Hof's avatar
Peter van 't Hof committed
245
      case b: String =>
246
        logger.warn("Value '" + any + "' is a string insteadof boolean in json file, trying auto convert")
247
        b.contains("true")
Peter van 't Hof's avatar
Peter van 't Hof committed
248
      case b: Int =>
249
        logger.warn("Value '" + any + "' is a int insteadof boolean in json file, trying auto convert")
Peter van 't Hof's avatar
Peter van 't Hof committed
250
        b > 0
251
252
253
254
      case _ => throw new IllegalStateException("Value '" + any + "' is not an boolean")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
255
  /** Convert Any to List[Any], fallback on list with 1 value */
256
257
258
  def any2list(any: Any): List[Any] = {
    if (any == null) return null
    any match {
259
      case l: List[_] => l
260
261
262
263
      case _          => List(any)
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
264
  /** Convert Any to List[String] */
265
  def any2stringList(any: Any): List[String] = {
266
    if (any == null) return null
267
    any2list(any).map(_.toString)
268
269
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
  /** Convert Any to List[Any], fallback on list with 1 value */
  def any2set(any: Any): Set[Any] = {
    if (any == null) return null
    any match {
      case s: Set[_]  => s.toSet
      case l: List[_] => l.toSet
      case _          => Set(any)
    }
  }

  /** Convert Any to List[String] */
  def any2stringSet(any: Any): Set[String] = {
    if (any == null) return null
    any2set(any).map(_.toString)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
286
  /** Convert Any to List[File] */
Peter van 't Hof's avatar
Peter van 't Hof committed
287
288
289
290
291
  def any2fileList(any: Any): List[File] = {
    if (any == null) return null
    any2list(any).map(x => new File(x.toString))
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
292
  /** Convert Any to Map[String, Any] */
293
294
295
  def any2map(any: Any): Map[String, Any] = {
    if (any == null) return null
    any match {
Peter van 't Hof's avatar
Peter van 't Hof committed
296
297
298
      case m: Map[_, _]                     => m.map(x => x._1.toString -> x._2)
      case m: java.util.LinkedHashMap[_, _] => nestedJavaHashMaptoScalaMap(m)
      case _                                => throw new IllegalStateException("Value '" + any + "' is not an Map")
299
300
301
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
302
303
304
305
306
307
308
309
310
311
  /** Convert nested java hash map to scala hash map */
  def nestedJavaHashMaptoScalaMap(input: java.util.LinkedHashMap[_, _]): Map[String, Any] = {
    input.map(value => {
      value._2 match {
        case m: java.util.LinkedHashMap[_, _] => value._1.toString -> nestedJavaHashMaptoScalaMap(m)
        case _                                => value._1.toString -> value._2
      }
    }).toMap
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
312
  /** Trait for implicit conversions for ConfigValue to native scala values */
Peter van 't Hof's avatar
Peter van 't Hof committed
313
  trait ImplicitConversions {
314
315
    import scala.language.implicitConversions

316
317
318
319
    private def requiredValue(value: ConfigValue): Boolean = {
      val exist = valueExists(value)
      if (!exist)
        BiopetQScript.addError("Value does not exist but is required, key: " + value.requestIndex.key +
320
          "  module: " + value.requestIndex.module,
Peter van 't Hof's avatar
Peter van 't Hof committed
321
          if (value.requestIndex.path != Nil) "  path: " + value.requestIndex.path.mkString("->") else null)
322
      exist
Peter van 't Hof's avatar
Peter van 't Hof committed
323
324
325
326
327
328
    }

    private def valueExists(value: ConfigValue): Boolean = {
      value != null && value.value != null && value.value != None
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
329
    /** Convert ConfigValue to File */
330
    implicit def configValue2file(value: ConfigValue): File = {
331
332
      if (requiredValue(value)) new File(any2string(value.value))
      else new File("")
333
    }
334

Peter van 't Hof's avatar
Peter van 't Hof committed
335
    /** Convert ConfigValue to File */
336
    implicit def configValue2optionFile(value: ConfigValue): Option[File] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
337
      if (valueExists(value)) Some(new File(any2string(value.value)))
338
      else None
339
340
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
341
    /** Convert ConfigValue to String */
342
    implicit def configValue2string(value: ConfigValue): String = {
343
344
      if (requiredValue(value)) any2string(value.value)
      else ""
345
    }
346

Peter van 't Hof's avatar
Peter van 't Hof committed
347
    /** Convert ConfigValue to String */
348
    implicit def configValue2optionString(value: ConfigValue): Option[String] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
349
      if (valueExists(value)) Some(any2string(value.value))
350
      else None
351
352
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
353
    /** Convert ConfigValue to Long */
354
    implicit def configValue2long(value: ConfigValue): Long = {
355
356
      if (requiredValue(value)) any2long(value.value)
      else 0L
357
    }
358

Peter van 't Hof's avatar
Peter van 't Hof committed
359
    /** Convert ConfigValue top Option[Long] */
360
    implicit def configValue2optionLong(value: ConfigValue): Option[Long] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
361
      if (valueExists(value)) Option(any2long(value.value))
362
      else None
363
    }
364

Peter van 't Hof's avatar
Peter van 't Hof committed
365
    /** Convert ConfigValue to Int */
366
    implicit def configValue2int(value: ConfigValue): Int = {
367
368
      if (requiredValue(value)) any2int(value.value)
      else 0
369
    }
370

Peter van 't Hof's avatar
Peter van 't Hof committed
371
    /** Convert ConfigValue to Option[Int] */
372
    implicit def configValue2optionInt(value: ConfigValue): Option[Int] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
373
      if (valueExists(value)) Option(any2int(value.value))
374
      else None
375
    }
376

Peter van 't Hof's avatar
Peter van 't Hof committed
377
    /** Convert ConfigValue to Double */
378
    implicit def configValue2double(value: ConfigValue): Double = {
379
380
      if (requiredValue(value)) any2double(value.value)
      else 0.0
381
    }
382

Peter van 't Hof's avatar
Peter van 't Hof committed
383
    /** Convert ConfigValue to Option[Double] */
384
    implicit def configValue2optionDouble(value: ConfigValue): Option[Double] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
385
      if (valueExists(value)) Option(any2double(value.value))
386
      else None
387
    }
388

Peter van 't Hof's avatar
Peter van 't Hof committed
389
    /** Convert ConfigValue to Float */
390
    implicit def configValue2float(value: ConfigValue): Float = {
391
392
      if (requiredValue(value)) any2float(value.value)
      else 0f
393
    }
394

Peter van 't Hof's avatar
Peter van 't Hof committed
395
    /** Convert ConfigValue to Option[Float] */
396
    implicit def configValue2optionFloat(value: ConfigValue): Option[Float] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
397
      if (valueExists(value)) Option(any2float(value.value))
398
      else None
399
    }
400

Peter van 't Hof's avatar
Peter van 't Hof committed
401
    /** Convert ConfigValue to Boolean */
402
    implicit def configValue2boolean(value: ConfigValue): Boolean = {
403
404
      if (requiredValue(value)) any2boolean(value.value)
      else false
405
    }
406

Peter van 't Hof's avatar
Peter van 't Hof committed
407
    /** Convert ConfigValue to Option[Boolean] */
408
    implicit def configValue2optionBoolean(value: ConfigValue): Option[Boolean] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
409
      if (valueExists(value)) Option(any2boolean(value.value))
410
      else None
411
    }
412

Peter van 't Hof's avatar
Peter van 't Hof committed
413
    /** Convert ConfigValue to List[Any] */
414
    implicit def configValue2list(value: ConfigValue): List[Any] = {
415
416
      if (requiredValue(value)) any2list(value.value)
      else Nil
417
    }
418

Peter van 't Hof's avatar
Peter van 't Hof committed
419
    /** Convert ConfigValue to List[String] */
420
    implicit def configValue2stringList(value: ConfigValue): List[String] = {
421
422
      if (requiredValue(value)) any2stringList(value.value)
      else Nil
423
    }
424

Peter van 't Hof's avatar
Peter van 't Hof committed
425
    /** Convert ConfigValue to List[File] */
Peter van 't Hof's avatar
Peter van 't Hof committed
426
427
428
429
430
    implicit def configValue2fileList(value: ConfigValue): List[File] = {
      if (requiredValue(value)) any2fileList(value.value)
      else Nil
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
431
    /** Convert ConfigValue to Set[String] */
432
    implicit def configValue2stringSet(value: ConfigValue): Set[String] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
433
      if (requiredValue(value)) any2stringSet(value.value)
434
      else Set()
435
    }
436

Peter van 't Hof's avatar
Peter van 't Hof committed
437
    /** Config config value to Map[String, Any] */
438
    implicit def configValue2map(value: ConfigValue): Map[String, Any] = {
439
440
      if (requiredValue(value)) any2map(value.value)
      else Map()
441
442
443
    }
  }
}