ConfigUtils.scala 16.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
16
17
18
package nl.lumc.sasc.biopet.utils

import java.io.File
Peter van 't Hof's avatar
Peter van 't Hof committed
19
import java.util
Peter van 't Hof's avatar
Peter van 't Hof committed
20
21
22

import argonaut.Argonaut._
import argonaut._
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import nl.lumc.sasc.biopet.utils.config.ConfigValue
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.yaml.snakeyaml.Yaml
Peter van 't Hof's avatar
Peter van 't Hof committed
25

Peter van 't Hof's avatar
Peter van 't Hof committed
26
import scala.collection.JavaConversions._
27

28
29
30
31
/**
 * This object contains general function for the config
 *
 */
32
object ConfigUtils extends Logging {
33
34
  /**
   * This method give back all nested values that does exist in map1 but not in map2
Peter van 't Hof's avatar
Peter van 't Hof committed
35
   *
36
37
38
39
   * @param map1 input map
   * @param map2 input map
   * @return Uniqe map1
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
40
  def uniqueKeys(map1: Map[String, Any], map2: Map[String, Any]): Map[String, Any] = {
41
42
    filterEmtpyMapValues(map1
      .flatMap {
Peter van 't Hof's avatar
Peter van 't Hof committed
43
        case (key, value: Map[_, _])             => Some(key -> uniqueKeys(value.asInstanceOf[Map[String, Any]], map2.getOrElse(key, Map()).asInstanceOf[Map[String, Any]]))
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
        case (key, value) if !map2.contains(key) => Some(key -> value)
        case _                                   => None
      })
  }

  /**
   * Filter values that are a map but are empty
   * @param map input map
   * @return output map
   */
  def filterEmtpyMapValues(map: Map[String, Any]): Map[String, Any] = {
    map.filter {
      case (key, value: Map[_, _]) => value.nonEmpty
      case _                       => true
    }
  }

61
62
63
64
65
66
  /**
   * Merge 2 maps, when value is in a map in map1 and map2 the value calls recursively this function
   * @param map1 Prio over map2
   * @param map2 Backup for map1
   * @return merged map
   */
67
68
  def mergeMaps(map1: Map[String, Any], map2: Map[String, Any],
                resolveConflict: (Any, Any, String) => Any = (m1, m2, key) => m1): Map[String, Any] = {
69
70
71
72
73
74
    var newMap: Map[String, Any] = Map()
    for (key <- map1.keySet.++(map2.keySet)) {
      if (!map2.contains(key)) newMap += (key -> map1(key))
      else if (!map1.contains(key)) newMap += (key -> map2(key))
      else {
        map1(key) match {
Peter van 't Hof's avatar
Peter van 't Hof committed
75
          case m1: Map[_, _] =>
76
            map2(key) match {
Peter van 't Hof's avatar
Peter van 't Hof committed
77
              case m2: Map[_, _] => newMap += (key -> mergeMaps(any2map(m1), any2map(m2), resolveConflict))
78
79
              case _             => newMap += (key -> map1(key))
            }
80
          case _ => newMap += (key -> resolveConflict(map1(key), map2(key), key))
81
82
83
        }
      }
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
84
    newMap
85
86
  }

87
88
89
90
91
92
  /**
   * Get nested map
   * @param map Map to search in
   * @param path Nested path to get from map
   * @return Nested map
   */
Peter van 't Hof's avatar
Peter van 't Hof committed
93
94
  def getMapFromPath(map: Map[String, Any], path: List[String]): Option[Map[String, Any]] = {
    val value = getValueFromPath(map, path) getOrElse { return None }
95
    value match {
Peter van 't Hof's avatar
Peter van 't Hof committed
96
      case m: Map[_, _] => Some(m.asInstanceOf[Map[String, Any]])
97
      case _            => throw new IllegalStateException("Value is not a map: " + value)
98
    }
99
100
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
101
102
103
104
105
106
  /**
   * Get nested value
   * @param map Map to search in
   * @param path Path to the value
   * @return Some(value) or None if not found
   */
107
  def getValueFromPath(map: Map[String, Any], path: List[String]): Option[Any] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
108
    val value = map.get(path.head)
Peter van 't Hof's avatar
Peter van 't Hof committed
109
    if (path.tail.isEmpty || value.isEmpty) value
Peter van 't Hof's avatar
Peter van 't Hof committed
110
    else value.get match {
Peter van 't Hof's avatar
Peter van 't Hof committed
111
112
113
      case map: Map[_, _]                     => getValueFromPath(map.asInstanceOf[Map[String, Any]], path.tail)
      case map: java.util.LinkedHashMap[_, _] => getValueFromPath(map.toMap.asInstanceOf[Map[String, Any]], path.tail)
      case _                                  => None
Peter van 't Hof's avatar
Peter van 't Hof committed
114
    }
115
116
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
117
  /** Make json aboject from a file */
118
  def fileToJson(configFile: File): Json = {
119
120
121
    logger.debug("Jsonfile: " + configFile)
    val jsonText = scala.io.Source.fromFile(configFile).mkString
    val json = Parse.parseOption(jsonText)
122
123
124
    logger.debug(json)

    json getOrElse {
125
126
      throw new IllegalStateException("The config JSON file is either not properly formatted or not a JSON file, file: " + configFile)
    }
127
128
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
129
  /** Convert config value to map */
130
  def fileToConfigMap(configFile: File): Map[String, Any] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
131
132

    val configMap = {
133
      if (configFile.getName.endsWith(".yaml") || configFile.getName.endsWith(".yml")) yamlToMap(configFile)
Peter van 't Hof's avatar
Peter van 't Hof committed
134
135
136
      else jsonToMap(fileToJson(configFile))
    }
    logger.debug("Contain: " + configMap)
Peter van 't Hof's avatar
Peter van 't Hof committed
137
    configMap
Peter van 't Hof's avatar
Peter van 't Hof committed
138
139
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
140
  /** Convert a yaml file to map[String, Any] */
Peter van 't Hof's avatar
Peter van 't Hof committed
141
142
143
144
  def yamlToMap(file: File): Map[String, Any] = {
    val yaml = new Yaml()
    val a = yaml.load(scala.io.Source.fromFile(file).reader())
    ConfigUtils.any2map(a)
145
146
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
147
  /** Convert json to native scala map/values */
148
149
150
151
152
153
154
  def jsonToMap(json: Json): Map[String, Any] = {
    var output: Map[String, Any] = Map()
    if (json.isObject) {
      for (key <- json.objectFieldsOrEmpty) {
        val value: Any = jsonToAny(json.field(key).get)
        output += (key -> value)
      }
155
    } else throw new IllegalStateException("Given value is no json object: " + json)
Peter van 't Hof's avatar
Peter van 't Hof committed
156
    output
157
158
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
159
  /** Convert json value to native scala value */
160
  def jsonToAny(json: Json): Any = {
Peter van 't Hof's avatar
Peter van 't Hof committed
161
    if (json.isObject) jsonToMap(json)
162
163
164
    else if (json.isArray) {
      var list: List[Any] = List()
      for (value <- json.array.get) list ::= jsonToAny(value)
Peter van 't Hof's avatar
Peter van 't Hof committed
165
166
167
      list.reverse
    } else if (json.isBool) json.bool.get
    else if (json.isString) json.string.get.toString
168
169
    else if (json.isNumber) {
      val num = json.number.get
Peter van 't Hof's avatar
Peter van 't Hof committed
170
171
172
      if (num % 1 > 0) num.toDouble
      else num.toLong
    } else if (json.isNull) None
173
    else throw new IllegalStateException("Config value type not supported, value: " + json)
174
175
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
176
  /** Convert native scala map to json */
177
178
179
180
181
182
183
184
185
  def mapToJson(map: Map[String, Any]): Json = {
    map.foldLeft(jEmptyObject)((acc, kv) => (kv._1 := {
      kv._2 match {
        case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
        case _            => anyToJson(kv._2)
      }
    }) ->: acc)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
186
  /** Convert native scala value to json, fall back on .toString if type is not a native scala value */
187
188
189
  def anyToJson(any: Any): Json = {
    any match {
      case j: Json      => j
Peter van 't Hof's avatar
Peter van 't Hof committed
190
      case None         => Json.jNull
191
      case Some(x)      => anyToJson(x)
192
      case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
Peter van 't Hof's avatar
Peter van 't Hof committed
193
      case l: List[_]   => Json.array(l.map(anyToJson): _*)
194
      case b: Boolean   => Json.jBool(b)
195
196
197
198
199
200
      case n: Int       => Json.jNumberOrString(n)
      case n: Double    => Json.jNumberOrString(n)
      case n: Long      => Json.jNumberOrString(n)
      case n: Short     => Json.jNumberOrString(n)
      case n: Float     => Json.jNumberOrString(n)
      case n: Byte      => Json.jNumberOrString(n)
Sander van der Zeeuw's avatar
Sander van der Zeeuw committed
201
      case null         => Json.jNull
202
203
204
205
      case _            => jString(any.toString)
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
206
  /** Convert Any to String */
207
208
209
  def any2string(any: Any): String = {
    if (any == null) return null
    any match {
210
211
      case s: String => s
      case _         => any.toString
212
213
214
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
215
  /** Convert Any to Int */
216
217
  def any2int(any: Any): Int = {
    any match {
218
      case i: Int    => i
219
      case i: Double => i.toInt
220
      case i: Long   => i.toInt
Peter van 't Hof's avatar
Peter van 't Hof committed
221
      case i: String =>
222
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
223
        i.toInt
224
225
226
227
      case _ => throw new IllegalStateException("Value '" + any + "' is not an int")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
228
  /** Convert Any to Long */
229
230
  def any2long(any: Any): Long = {
    any match {
231
232
233
      case l: Double => l.toLong
      case l: Int    => l.toLong
      case l: Long   => l
Peter van 't Hof's avatar
Peter van 't Hof committed
234
      case l: String =>
235
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
236
        l.toLong
237
238
239
240
      case _ => throw new IllegalStateException("Value '" + any + "' is not an int")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
241
  /** Convert Any to Double */
242
243
  def any2double(any: Any): Double = {
    any match {
244
245
246
      case d: Double => d
      case d: Float  => d.toDouble
      case d: Int    => d.toDouble
247
      case f: Long   => f.toDouble
Peter van 't Hof's avatar
Peter van 't Hof committed
248
      case d: String =>
249
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
Peter van 't Hof's avatar
Peter van 't Hof committed
250
        d.toDouble
251
      case _ => throw new IllegalStateException("Value '" + any + "' is not an number")
252
253
254
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
255
  /** Convert Any to Float */
256
257
  def any2float(any: Any): Float = {
    any match {
258
259
      case f: Double => f.toFloat
      case f: Int    => f.toFloat
260
      case f: Long   => f.toFloat
261
      case f: Float  => f
Peter van 't Hof's avatar
Peter van 't Hof committed
262
      case f: String =>
263
        logger.warn("Value '" + any + "' is a string insteadof int in json file, trying auto convert")
264
        f.toFloat
265
      case _ => throw new IllegalStateException("Value '" + any + "' is not an number")
266
267
268
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
269
  /** Convert Any to Boolean */
270
271
  def any2boolean(any: Any): Boolean = {
    any match {
272
      case b: Boolean => b
Peter van 't Hof's avatar
Peter van 't Hof committed
273
      case b: String =>
274
        logger.warn("Value '" + any + "' is a string insteadof boolean in json file, trying auto convert")
275
        b.contains("true")
Peter van 't Hof's avatar
Peter van 't Hof committed
276
      case b: Int =>
277
        logger.warn("Value '" + any + "' is a int insteadof boolean in json file, trying auto convert")
Peter van 't Hof's avatar
Peter van 't Hof committed
278
        b > 0
279
280
281
282
      case _ => throw new IllegalStateException("Value '" + any + "' is not an boolean")
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
283
  /** Convert Any to List[Any], fallback on list with 1 value */
284
285
286
  def any2list(any: Any): List[Any] = {
    if (any == null) return null
    any match {
Peter van 't Hof's avatar
Peter van 't Hof committed
287
      case l: List[_]           => l
Peter van 't Hof's avatar
Peter van 't Hof committed
288
      case l: util.ArrayList[_] => l.toList
Peter van 't Hof's avatar
Peter van 't Hof committed
289
      case _                    => List(any)
290
291
292
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
293
  /** Convert Any to List[String] */
294
  def any2stringList(any: Any): List[String] = {
295
    if (any == null) return null
296
    any2list(any).map(_.toString)
297
298
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
299
300
301
302
  /** Convert Any to List[Any], fallback on list with 1 value */
  def any2set(any: Any): Set[Any] = {
    if (any == null) return null
    any match {
Peter van 't Hof's avatar
Peter van 't Hof committed
303
304
305
306
      case s: Set[_]            => s.toSet
      case l: List[_]           => l.toSet
      case l: util.ArrayList[_] => l.toSet
      case _                    => Set(any)
Peter van 't Hof's avatar
Peter van 't Hof committed
307
308
309
310
311
312
313
314
315
    }
  }

  /** Convert Any to List[String] */
  def any2stringSet(any: Any): Set[String] = {
    if (any == null) return null
    any2set(any).map(_.toString)
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
316
  /** Convert Any to List[File] */
Peter van 't Hof's avatar
Peter van 't Hof committed
317
318
319
320
321
  def any2fileList(any: Any): List[File] = {
    if (any == null) return null
    any2list(any).map(x => new File(x.toString))
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
322
  /** Convert Any to Map[String, Any] */
323
324
325
  def any2map(any: Any): Map[String, Any] = {
    if (any == null) return null
    any match {
Peter van 't Hof's avatar
Peter van 't Hof committed
326
327
328
      case m: Map[_, _]                     => m.map(x => x._1.toString -> x._2)
      case m: java.util.LinkedHashMap[_, _] => nestedJavaHashMaptoScalaMap(m)
      case _                                => throw new IllegalStateException("Value '" + any + "' is not an Map")
329
330
331
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
332
333
334
335
336
337
338
339
340
341
  /** Convert nested java hash map to scala hash map */
  def nestedJavaHashMaptoScalaMap(input: java.util.LinkedHashMap[_, _]): Map[String, Any] = {
    input.map(value => {
      value._2 match {
        case m: java.util.LinkedHashMap[_, _] => value._1.toString -> nestedJavaHashMaptoScalaMap(m)
        case _                                => value._1.toString -> value._2
      }
    }).toMap
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
342
  /** Trait for implicit conversions for ConfigValue to native scala values */
Peter van 't Hof's avatar
Peter van 't Hof committed
343
  trait ImplicitConversions {
344
345
    import scala.language.implicitConversions

346
347
348
    private def requiredValue(value: ConfigValue): Boolean = {
      val exist = valueExists(value)
      if (!exist)
Peter van 't Hof's avatar
Peter van 't Hof committed
349
        Logging.addError("Value does not exist but is required, key: " + value.requestIndex.key +
Sander Bollen's avatar
Sander Bollen committed
350
          "  namespace: " + value.requestIndex.module,
Peter van 't Hof's avatar
Peter van 't Hof committed
351
          if (value.requestIndex.path != Nil) "  path: " + value.requestIndex.path.mkString("->") else null)
352
      exist
Peter van 't Hof's avatar
Peter van 't Hof committed
353
354
355
356
357
358
    }

    private def valueExists(value: ConfigValue): Boolean = {
      value != null && value.value != null && value.value != None
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
359
    /** Convert ConfigValue to File */
360
    implicit def configValue2file(value: ConfigValue): File = {
361
362
      if (requiredValue(value)) new File(any2string(value.value))
      else new File("")
363
    }
364

Peter van 't Hof's avatar
Peter van 't Hof committed
365
    /** Convert ConfigValue to File */
366
    implicit def configValue2optionFile(value: ConfigValue): Option[File] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
367
      if (valueExists(value)) Some(new File(any2string(value.value)))
368
      else None
369
370
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
371
    /** Convert ConfigValue to String */
372
    implicit def configValue2string(value: ConfigValue): String = {
373
374
      if (requiredValue(value)) any2string(value.value)
      else ""
375
    }
376

Peter van 't Hof's avatar
Peter van 't Hof committed
377
    /** Convert ConfigValue to String */
378
    implicit def configValue2optionString(value: ConfigValue): Option[String] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
379
      if (valueExists(value)) Some(any2string(value.value))
380
      else None
381
382
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
383
    /** Convert ConfigValue to Long */
384
    implicit def configValue2long(value: ConfigValue): Long = {
385
386
      if (requiredValue(value)) any2long(value.value)
      else 0L
387
    }
388

Peter van 't Hof's avatar
Peter van 't Hof committed
389
    /** Convert ConfigValue top Option[Long] */
390
    implicit def configValue2optionLong(value: ConfigValue): Option[Long] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
391
      if (valueExists(value)) Option(any2long(value.value))
392
      else None
393
    }
394

Peter van 't Hof's avatar
Peter van 't Hof committed
395
    /** Convert ConfigValue to Int */
396
    implicit def configValue2int(value: ConfigValue): Int = {
397
398
      if (requiredValue(value)) any2int(value.value)
      else 0
399
    }
400

Peter van 't Hof's avatar
Peter van 't Hof committed
401
    /** Convert ConfigValue to Option[Int] */
402
    implicit def configValue2optionInt(value: ConfigValue): Option[Int] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
403
      if (valueExists(value)) Option(any2int(value.value))
404
      else None
405
    }
406

Peter van 't Hof's avatar
Peter van 't Hof committed
407
    /** Convert ConfigValue to Double */
408
    implicit def configValue2double(value: ConfigValue): Double = {
409
410
      if (requiredValue(value)) any2double(value.value)
      else 0.0
411
    }
412

Peter van 't Hof's avatar
Peter van 't Hof committed
413
    /** Convert ConfigValue to Option[Double] */
414
    implicit def configValue2optionDouble(value: ConfigValue): Option[Double] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
415
      if (valueExists(value)) Option(any2double(value.value))
416
      else None
417
    }
418

Peter van 't Hof's avatar
Peter van 't Hof committed
419
    /** Convert ConfigValue to Float */
420
    implicit def configValue2float(value: ConfigValue): Float = {
421
422
      if (requiredValue(value)) any2float(value.value)
      else 0f
423
    }
424

Peter van 't Hof's avatar
Peter van 't Hof committed
425
    /** Convert ConfigValue to Option[Float] */
426
    implicit def configValue2optionFloat(value: ConfigValue): Option[Float] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
427
      if (valueExists(value)) Option(any2float(value.value))
428
      else None
429
    }
430

Peter van 't Hof's avatar
Peter van 't Hof committed
431
    /** Convert ConfigValue to Boolean */
432
    implicit def configValue2boolean(value: ConfigValue): Boolean = {
433
434
      if (requiredValue(value)) any2boolean(value.value)
      else false
435
    }
436

Peter van 't Hof's avatar
Peter van 't Hof committed
437
    /** Convert ConfigValue to Option[Boolean] */
438
    implicit def configValue2optionBoolean(value: ConfigValue): Option[Boolean] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
439
      if (valueExists(value)) Option(any2boolean(value.value))
440
      else None
441
    }
442

Peter van 't Hof's avatar
Peter van 't Hof committed
443
    /** Convert ConfigValue to List[Any] */
444
    implicit def configValue2list(value: ConfigValue): List[Any] = {
445
446
      if (requiredValue(value)) any2list(value.value)
      else Nil
447
    }
448

Peter van 't Hof's avatar
Peter van 't Hof committed
449
    /** Convert ConfigValue to List[String] */
450
    implicit def configValue2stringList(value: ConfigValue): List[String] = {
451
452
      if (requiredValue(value)) any2stringList(value.value)
      else Nil
453
    }
454

Peter van 't Hof's avatar
Peter van 't Hof committed
455
    /** Convert ConfigValue to List[File] */
Peter van 't Hof's avatar
Peter van 't Hof committed
456
457
458
459
460
    implicit def configValue2fileList(value: ConfigValue): List[File] = {
      if (requiredValue(value)) any2fileList(value.value)
      else Nil
    }

461
462
463
464
465
466
467
468
469
470
471
472
    /** Convert ConfigValue to List[Double] */
    implicit def configValue2doubleList(value: ConfigValue): List[Double] = {
      if (requiredValue(value)) any2list(value.value).map(any2double(_))
      else Nil
    }

    /** Convert ConfigValue to List[Int] */
    implicit def configValue2intList(value: ConfigValue): List[Int] = {
      if (requiredValue(value)) any2list(value.value).map(any2int(_))
      else Nil
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
473
    /** Convert ConfigValue to Set[String] */
474
    implicit def configValue2stringSet(value: ConfigValue): Set[String] = {
Peter van 't Hof's avatar
Peter van 't Hof committed
475
      if (requiredValue(value)) any2stringSet(value.value)
476
      else Set()
477
    }
478

Peter van 't Hof's avatar
Peter van 't Hof committed
479
    /** Config config value to Map[String, Any] */
480
    implicit def configValue2map(value: ConfigValue): Map[String, Any] = {
481
482
      if (requiredValue(value)) any2map(value.value)
      else Map()
483
484
485
    }
  }
}