Commit fe662220 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Merge branch 'develop' into feature-FilterOnMappingQualityCarp

# Conflicts:
#	public/carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala
parents ce8ecffb aca99bf8
......@@ -153,11 +153,11 @@ class GatkVariantcalling(val root: Configurable) extends QScript with BiopetQScr
scriptOutput.rawVcfFile = m2v.output
val vcfFilter = new VcfFilter(this) {
override def defaults = ConfigUtils.mergeMaps(Map("min_sample_depth" -> 8,
override def defaults = Map("min_sample_depth" -> 8,
"min_alternate_depth" -> 2,
"min_samples_pass" -> 1,
"filter_ref_calls" -> true
), super.defaults)
)
}
vcfFilter.inputVcf = m2v.output
vcfFilter.outputVcf = swapExt(outputDir, m2v.output, ".vcf", ".filter.vcf.gz")
......
......@@ -35,10 +35,10 @@ trait BastyTrait extends MultiSampleQScript {
def variantcallers = List("freebayes")
override def defaults = ConfigUtils.mergeMaps(Map(
override def defaults = Map(
"ploidy" -> 1,
"variantcallers" -> variantcallers
), super.defaults)
)
lazy val shiva: ShivaTrait = new Shiva(qscript)
......
......@@ -47,7 +47,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction with Refe
override def defaultCoreMemory = 4.0
override def defaultThreads = 8
var sam: Boolean = config("sam", default = true)
var sam: Boolean = config("sam", default = false)
var sam_RG: Option[String] = config("sam-RG")
var seedlen: Option[Int] = config("seedlen")
var seedmms: Option[Int] = config("seedmms")
......
......@@ -53,6 +53,8 @@ class CollectGcBiasMetrics(val root: Configurable) extends Picard with Summariza
@Argument(doc = "IS_BISULFITE_SEQUENCED", required = false)
var isBisulfiteSequinced: Option[Boolean] = config("isbisulfitesequinced")
override def defaultCoreMemory = 8.0
override def beforeGraph() {
super.beforeGraph()
if (outputChart == null) outputChart = new File(output + ".pdf")
......
......@@ -33,7 +33,7 @@ class CollectMultipleMetrics(val root: Configurable) extends Picard with Summari
javaMainClass = new picard.analysis.CollectMultipleMetrics().getClass.getName
override def defaultCoreMemory = 6.0
override def defaultCoreMemory = 8.0
@Input(doc = "The input SAM or BAM files to analyze", required = true)
var input: File = null
......
......@@ -25,17 +25,10 @@ import scala.sys.process.{ Process, ProcessLogger }
class GsnapTest extends TestNGSuite with Matchers {
private def setConfig(key: String, value: String): Map[String, Any] = {
val oldMap: Map[String, Any] = Config.global.map
Config.global.map += (key -> value)
oldMap
}
private def restoreConfig(oldMap: Map[String, Any]): Unit = Config.global.map = oldMap
@BeforeClass def checkExecutable() = {
val oldMap = setConfig("db", "mock")
val wrapper = new Gsnap(null)
val wrapper = new Gsnap(null) {
override def globalConfig = new Config(Map("db" -> "mock"))
}
val proc = Process(wrapper.versionCommand)
val exitCode =
try {
......@@ -47,13 +40,12 @@ class GsnapTest extends TestNGSuite with Matchers {
}
if (exitCode != 0)
throw new SkipException("Skipping GSNAP test because the executable can not be found")
restoreConfig(oldMap)
}
@Test(description = "GSNAP version number capture from executable")
def testVersion() = {
val oldMap = setConfig("db", "mock")
new Gsnap(null).getVersion should not be "N/A"
restoreConfig(oldMap)
new Gsnap(null) {
override def globalConfig = new Config(Map("db" -> "mock"))
}.getVersion should not be "N/A"
}
}
......@@ -21,11 +21,11 @@ import nl.lumc.sasc.biopet.utils.ConfigUtils._
/**
* This class can store nested config values
* @param map Map with value for new config
* @param _map Map with value for new config
* @constructor Load config with existing map
*/
class Config(var map: Map[String, Any],
protected[config] var defaults: Map[String, Any] = Map()) extends Logging {
class Config(protected var _map: Map[String, Any],
protected var _defaults: Map[String, Any] = Map()) extends Logging {
logger.debug("Init phase of config")
/** Default constructor */
......@@ -34,6 +34,9 @@ class Config(var map: Map[String, Any],
loadDefaultConfig()
}
def map = _map
def defaults = _defaults
/**
* Loading a environmental variable as location of config files to merge into the config
* @param valueName Name of value
......@@ -65,13 +68,13 @@ class Config(var map: Map[String, Any],
def loadConfigFile(configFile: File, default: Boolean = false) {
val configMap = fileToConfigMap(configFile)
if (default) {
if (defaults.isEmpty) defaults = configMap
else defaults = mergeMaps(configMap, defaults)
logger.debug("New defaults: " + defaults)
if (_defaults.isEmpty) _defaults = configMap
else _defaults = mergeMaps(configMap, _defaults)
logger.debug("New defaults: " + _defaults)
} else {
if (map.isEmpty) map = configMap
else map = mergeMaps(configMap, map)
logger.debug("New config: " + map)
if (_map.isEmpty) _map = configMap
else _map = mergeMaps(configMap, _map)
logger.debug("New config: " + _map)
}
}
......@@ -84,11 +87,12 @@ class Config(var map: Map[String, Any],
*/
def addValue(key: String, value: Any, path: List[String] = Nil, default: Boolean = false): Unit = {
val valueMap = path.foldRight(Map(key -> value))((a, b) => Map(a -> b))
if (default) defaults = mergeMaps(valueMap, defaults)
else map = mergeMaps(valueMap, map)
if (default) _defaults = mergeMaps(valueMap, _defaults)
else _map = mergeMaps(valueMap, _map)
}
protected[config] var notFoundCache: List[ConfigValueIndex] = List()
protected[config] var fixedCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] var foundCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] var defaultCache: Map[ConfigValueIndex, ConfigValue] = Map()
protected[config] def clearCache(): Unit = {
......@@ -103,24 +107,39 @@ class Config(var map: Map[String, Any],
* @param s key
* @return True if exist
*/
def contains(s: String): Boolean = map.contains(s)
def contains(s: String): Boolean = _map.contains(s)
/**
* Checks if value exist in config
* @param requestedIndex Index to value
* @return True if exist
*/
def contains(requestedIndex: ConfigValueIndex): Boolean = contains(requestedIndex, Map())
/**
* Checks if value exist in config
* @param requestedIndex Index to value
* @param fixedValues Fixed values
* @return True if exist
*/
def contains(requestedIndex: ConfigValueIndex): Boolean =
def contains(requestedIndex: ConfigValueIndex, fixedValues: Map[String, Any]): Boolean =
if (notFoundCache.contains(requestedIndex)) false
else if (fixedCache.contains(requestedIndex)) true
else if (foundCache.contains(requestedIndex)) true
else {
val value = Config.getValueFromMap(map, requestedIndex)
if (value.isDefined && value.get.value != None) {
foundCache += (requestedIndex -> value.get)
val fixedValue = Config.getValueFromMap(fixedValues, requestedIndex)
if (fixedValue.isDefined) {
fixedCache += (requestedIndex -> fixedValue.get)
true
} else {
notFoundCache +:= requestedIndex
false
val value = Config.getValueFromMap(_map, requestedIndex)
if (value.isDefined && value.get.value != None) {
foundCache += (requestedIndex -> value.get)
true
} else {
notFoundCache +:= requestedIndex
false
}
}
}
......@@ -132,9 +151,12 @@ class Config(var map: Map[String, Any],
* @param freeVar Default true, if set false value must exist in module
* @return True if exist
*/
def contains(module: String, path: List[String], key: String, freeVar: Boolean = true): Boolean = {
def contains(module: String, path: List[String],
key: String,
freeVar: Boolean = true,
fixedValues: Map[String, Any] = Map()): Boolean = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
contains(requestedIndex)
contains(requestedIndex, fixedValues)
}
/**
......@@ -146,10 +168,23 @@ class Config(var map: Map[String, Any],
* @param freeVar Default true, if set false value must exist in module
* @return Config value
*/
protected[config] def apply(module: String, path: List[String], key: String, default: Any = null, freeVar: Boolean = true): ConfigValue = {
protected[config] def apply(module: String,
path: List[String],
key: String,
default: Any = null,
freeVar: Boolean = true,
fixedValues: Map[String, Any] = Map()): ConfigValue = {
val requestedIndex = ConfigValueIndex(module, path, key, freeVar)
if (contains(requestedIndex)) foundCache(requestedIndex)
else if (default != null) {
if (contains(requestedIndex, fixedValues)) {
val fixedValue = fixedCache.get(requestedIndex)
if (fixedValue.isDefined) {
val userValue = Config.getValueFromMap(_map, requestedIndex)
if (userValue.isDefined)
logger.warn(s"Ignoring user-supplied value ${requestedIndex.key} at path ${requestedIndex.path} because it is a fixed value.")
}
fixedValue.getOrElse(foundCache(requestedIndex))
} else if (default != null) {
defaultCache += (requestedIndex -> ConfigValue(requestedIndex, null, default, freeVar))
defaultCache(requestedIndex)
} else ConfigValue(requestedIndex, null, null, freeVar)
......@@ -179,9 +214,11 @@ class Config(var map: Map[String, Any],
// Positions where values are found
val found = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
val fixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.foundIndex, x._2.value)))
// Positions where to start searching
val effectiveFound = convertIndexValuesToMap(foundCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val effectiveFixed = convertIndexValuesToMap(fixedCache.filter(!_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val effectiveDefaultFound = convertIndexValuesToMap(defaultCache.filter(_._2.default).toList.map(x => (x._2.requestIndex, x._2.value)), Some(false))
val notFound = convertIndexValuesToMap(notFoundCache.map((_, None)), Some(false))
......@@ -189,16 +226,19 @@ class Config(var map: Map[String, Any],
val fullEffective = ConfigUtils.mergeMaps(effectiveFound, effectiveDefaultFound)
val fullEffectiveWithNotFound = ConfigUtils.mergeMaps(fullEffective, notFound)
writeMapToJsonFile(this.map, "input")
writeMapToJsonFile(_map, "input")
writeMapToJsonFile(_defaults, "defaults")
writeMapToJsonFile(found, "found")
writeMapToJsonFile(fixed, "fixed")
writeMapToJsonFile(effectiveFound, "effective.found")
writeMapToJsonFile(effectiveFixed, "effective.fixed")
writeMapToJsonFile(effectiveDefaultFound, "effective.defaults")
writeMapToJsonFile(notFound, "not.found")
writeMapToJsonFile(fullEffective, "effective.full")
writeMapToJsonFile(fullEffectiveWithNotFound, "effective.full.notfound")
}
override def toString: String = map.toString()
override def toString: String = _map.toString()
}
object Config extends Logging {
......@@ -210,7 +250,7 @@ object Config extends Logging {
* @param config2 Low prio map
* @return Merged config
*/
def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1.map, config2.map))
def mergeConfigs(config1: Config, config2: Config): Config = new Config(mergeMaps(config1._map, config2._map))
/**
* Search for value in index position in a map
......
......@@ -15,6 +15,7 @@
*/
package nl.lumc.sasc.biopet.utils.config
import nl.lumc.sasc.biopet.utils.ConfigUtils
import nl.lumc.sasc.biopet.utils.ConfigUtils.ImplicitConversions
trait Configurable extends ImplicitConversions {
......@@ -35,9 +36,28 @@ trait Configurable extends ImplicitConversions {
def configFullPath: List[String] = configPath ::: configName :: Nil
/** Map to store defaults for config */
def defaults: Map[String, Any] = {
if (root != null) root.defaults
else globalConfig.defaults
def defaults: Map[String, Any] = Map()
/** This method merge defaults from the root to it's own */
protected def internalDefaults: Map[String, Any] = {
(root != null, defaults.isEmpty) match {
case (true, true) => root.defaults
case (true, false) => ConfigUtils.mergeMaps(defaults, root.defaults)
case (false, true) => globalConfig.defaults
case (false, false) => ConfigUtils.mergeMaps(defaults, globalConfig.defaults)
}
}
/** All values found in this map will be skipped from the user config */
def fixedValues: Map[String, Any] = Map()
/** This method merge fixedValues from the root to it's own */
protected def internalFixedValues: Map[String, Any] = {
(root != null, fixedValues.isEmpty) match {
case (true, true) => root.internalFixedValues
case (true, false) => ConfigUtils.mergeMaps(fixedValues, root.internalFixedValues)
case _ => fixedValues
}
}
val config = new ConfigFunctions
......@@ -90,11 +110,11 @@ trait Configurable extends ImplicitConversions {
val m = if (submodule != null) submodule else configName
val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
val d = {
val value = Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar))
val value = Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar))
if (value.isDefined) value.get.value else default
}
if (d == null) globalConfig(m, p, key, freeVar = freeVar)
else globalConfig(m, p, key, d, freeVar)
if (d == null) globalConfig(m, p, key, freeVar = freeVar, fixedValues = internalFixedValues)
else globalConfig(m, p, key, d, freeVar, fixedValues = internalFixedValues)
}
/**
......@@ -117,7 +137,7 @@ trait Configurable extends ImplicitConversions {
val m = if (submodule != null) submodule else configName
val p = if (path == null) getConfigPath(s, l, submodule) ::: subPath else path
globalConfig.contains(m, p, key, freeVar) || Config.getValueFromMap(defaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
globalConfig.contains(m, p, key, freeVar, internalFixedValues) || Config.getValueFromMap(internalDefaults, ConfigValueIndex(m, p, key, freeVar)).isDefined
}
}
}
......@@ -25,10 +25,36 @@ import org.testng.annotations.Test
* Created by pjvan_thof on 1/8/15.
*/
class ConfigurableTest extends TestNGSuite with Matchers {
abstract class Cfg extends Configurable {
def get(key: String,
default: String = null,
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null) = {
config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
}
}
class ClassA(val root: Configurable) extends Cfg
class ClassB(val root: Configurable) extends Cfg {
lazy val classA = new ClassA(this)
// Why this needs to be lazy?
}
class ClassC(val root: Configurable) extends Cfg {
def this() = this(null)
lazy val classB = new ClassB(this)
// Why this needs to be lazy?
}
@Test def testConfigurable(): Unit = {
val classC = new ClassC {
override def configName = "classc"
override val globalConfig = new Config(ConfigurableTest.map)
override val fixedValues = Map("fixed" -> "fixed")
}
classC.configPath shouldBe Nil
......@@ -51,46 +77,33 @@ class ConfigurableTest extends TestNGSuite with Matchers {
classC.get("bla", sample = "sample1", library = "library1").asString shouldBe "bla"
classC.get("test", sample = "sample1", library = "library1").asString shouldBe "test"
classC.get("test", sample = "sample1").asString shouldBe "test"
}
}
abstract class Cfg extends Configurable {
def get(key: String,
default: String = null,
submodule: String = null,
freeVar: Boolean = true,
sample: String = null,
library: String = null) = {
config(key, default, submodule, freeVar = freeVar, sample = sample, library = library)
// Fixed values
classC.get("fixed").asString shouldBe "fixed"
classC.classB.get("fixed").asString shouldBe "fixed"
classC.classB.classA.get("fixed").asString shouldBe "fixed"
}
}
class ClassA(val root: Configurable) extends Cfg
class ClassB(val root: Configurable) extends Cfg {
lazy val classA = new ClassA(this)
// Why this needs to be lazy?
}
class ClassC(val root: Configurable) extends Cfg {
def this() = this(null)
lazy val classB = new ClassB(this)
// Why this needs to be lazy?
}
object ConfigurableTest {
val map = Map(
"fixed" -> "nonfixed",
"classa" -> Map(
"k1" -> "a1"
"k1" -> "a1",
"fixed" -> "nonfixed"
), "classb" -> Map(
"k1" -> "b1"
"k1" -> "b1",
"fixed" -> "nonfixed"
), "classc" -> Map(
"k1" -> "c1"
"k1" -> "c1",
"fixed" -> "nonfixed"
), "samples" -> Map(
"sample1" -> Map(
"fixed" -> "nonfixed",
"test" -> "test",
"libraries" -> Map(
"library1" -> Map(
"fixed" -> "nonfixed",
"bla" -> "bla"
)
)
......
......@@ -39,16 +39,14 @@ class Carp(val root: Configurable) extends QScript with MultiSampleQScript with
qscript =>
def this() = this(null)
override def defaults = ConfigUtils.mergeMaps(Map(
override def defaults = Map(
"mapping" -> Map(
"skip_markduplicates" -> false,
"aligner" -> "bwa-mem"
),
"samtoolsview" -> Map("q" -> 10)
), super.defaults)
)
def summaryFile = new File(outputDir, "Carp.summary.json")
//TODO: Add summary
......
......@@ -201,7 +201,8 @@ class Fastqc(root: Configurable) extends nl.lumc.sasc.biopet.extensions.Fastqc(r
def summaryStats: Map[String, Any] = Map(
"per_base_sequence_quality" -> perBaseSequenceQuality,
"per_base_sequence_content" -> perBaseSequenceContent)
"per_base_sequence_content" -> perBaseSequenceContent,
"adapters" -> foundAdapters.map(x => x.name -> x.seq).toMap)
}
object Fastqc {
......
......@@ -250,24 +250,22 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
if (fastq_R1.length != fastq_R2.length && paired)
throw new IllegalStateException("R1 and R2 file number is not the same")
if (!skipTrim || !skipClip) {
if (fastq_R1.length > 1) {
add(Zcat(this, fastq_R1, fastqR1Qc) | new Gzip(this) > fastqR1Qc)
if (paired) add(Zcat(this, fastq_R2, fastqR2Qc.get) | new Gzip(this) > fastqR2Qc.get)
}
if (fastq_R1.length > 1) {
add(Zcat(this, fastq_R1, fastqR1Qc) | new Gzip(this) > fastqR1Qc)
if (paired) add(Zcat(this, fastq_R2, fastqR2Qc.get) | new Gzip(this) > fastqR2Qc.get)
}
outputFiles += ("output_R1_gzip" -> fastqR1Qc)
if (paired) outputFiles += ("output_R2_gzip" -> fastqR2Qc.get)
outputFiles += ("output_R1_gzip" -> fastqR1Qc)
if (paired) outputFiles += ("output_R2_gzip" -> fastqR2Qc.get)
fastqc_R1_after = Fastqc(this, fastqR1Qc, new File(outputDir, R1_name + ".qc.fastqc/"))
add(fastqc_R1_after)
addSummarizable(fastqc_R1_after, "fastqc_R1_qc")
fastqc_R1_after = Fastqc(this, fastqR1Qc, new File(outputDir, R1_name + ".qc.fastqc/"))
add(fastqc_R1_after)
addSummarizable(fastqc_R1_after, "fastqc_R1_qc")
if (paired) {
fastqc_R2_after = Fastqc(this, fastqR2Qc.get, new File(outputDir, R2_name + ".qc.fastqc/"))
add(fastqc_R2_after)
addSummarizable(fastqc_R2_after, "fastqc_R2_qc")
}
if (paired) {
fastqc_R2_after = Fastqc(this, fastqR2Qc.get, new File(outputDir, R2_name + ".qc.fastqc/"))
add(fastqc_R2_after)
addSummarizable(fastqc_R2_after, "fastqc_R2_qc")
}
addSummaryJobs()
......
......@@ -73,11 +73,7 @@ class FlexiprepTest extends TestNGSuite with Matchers {
flexiprep.libId = Some("1")
flexiprep.script()
flexiprep.functions.count(_.isInstanceOf[Fastqc]) shouldBe (
if (paired && (skipClip && skipTrim)) 2
else if (!paired && (skipClip && skipTrim)) 1
else if (paired && !(skipClip && skipTrim)) 4
else if (!paired && !(skipClip && skipTrim)) 2)
flexiprep.functions.count(_.isInstanceOf[Fastqc]) shouldBe (if (paired) 4 else 2)
flexiprep.functions.count(_.isInstanceOf[SeqStat]) shouldBe (if (paired) 4 else 2)
}
......
......@@ -100,24 +100,23 @@ class Gentrap(val root: Configurable) extends QScript
})
/** Default pipeline config */
override def defaults = ConfigUtils.mergeMaps(
Map(
"gsnap" -> Map(
"novelsplicing" -> 1,
"batch" -> 4,
"format" -> "sam"
),
"cutadapt" -> Map("minimum_length" -> 20),
// avoid conflicts when merging since the MarkDuplicate tags often cause merges to fail
"picard" -> Map(
"programrecordid" -> "null"
),
// disable markduplicates since it may not play well with all aligners (this can still be overriden via config)
"mapping" -> Map(
"skip_markduplicates" -> true,
"skip_metrics" -> true
)