Commit 3009ebc0 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Merge remote-tracking branch 'remotes/origin/develop' into feature-fixStarOptions

parents d6fcf3f9 8a322f5c
......@@ -28,6 +28,7 @@
<module>../public/bam2wig</module>
<module>../public/carp</module>
<module>../public/toucan</module>
<module>../public/gwas-test</module>
<module>../public/shiva</module>
<module>../public/basty</module>
<module>../public/tinycap</module>
......
......@@ -138,7 +138,7 @@ Setting up the pipeline is done within the pipeline itself, fine-tuning is alway
For our new pipeline, one should setup the (default) config options.
Since our pipeline is called `HelloPipeline`, the root of the configoptions will called `hellopipeline` (lowercaps).
Since our pipeline is called `HelloPipeline`, the root of the namespace for our pipeline will be called `hellopipeline` (lowercaps).
```json
{
......
......@@ -20,7 +20,7 @@ We assume here that the cluster will amplify those values by the number of threa
- **residentFactor**: 1.2
- **vmemFactor**: 1.4, 2.0 for java jobs
This are de defaults of biopet but each extension in biopet can set their own defaults. As example the *bwa mem* tools
This are the defaults of biopet but each extension in biopet can set their own defaults. As example the *bwa mem* tools
use by default 8 `threads` and `core_memory` of 6.0.
### Config
......
......@@ -34,7 +34,7 @@ Please refer [to our mapping pipeline](mapping.md) for information about how the
#### Required configuration values
| Submodule | Name | Type | Default | Function |
| namespace | Name | Type | Default | Function |
| --------- | ---- | ---- | ------- | -------- |
| shiva | variantcallers | List[String] | | Which variant caller to use |
| - | output_dir | Path | Path to output directory |
......@@ -44,7 +44,7 @@ Please refer [to our mapping pipeline](mapping.md) for information about how the
Specific configuration options additional to Basty are:
| Submodule | Name | Type | Default | Function |
| namespace | Name | Type | Default | Function |
| --------- | ---- | ---- | ------- | -------- |
| raxml | seed | Integer | 12345 | RAxML Random seed|
| raxml | raxml_ml_model | String | GTRGAMMAX | RAxML model |
......
......@@ -105,7 +105,7 @@ To view all possible config options please navigate to our Gitlab wiki page
<a href="https://git.lumc.nl/biopet/biopet/wikis/GATK-Variantcalling-Pipeline" target="_blank">Config</a>
### Required settings
| Namespace | Name | Type | Default | Function |
| Confignamespace | Name | Type | Default | Function |
| ----------- | ---- | ---- | ------- | -------- |
| - | output_dir | String | | Path to output directory |
| Shiva | variantcallers | List[String] | | Which variant callers to use |
......@@ -113,7 +113,7 @@ To view all possible config options please navigate to our Gitlab wiki page
### Config options
| Namespace | Name | Type | Default | Function |
| ConfignNamespace | Name | Type | Default | Function |
| ----------- | ---- | ----- | ------- | -------- |
| shiva | species | String | unknown_species | Name of species, like H.sapiens |
| shiva | reference_name | String | unknown_reference_name | Name of reference, like hg19 |
......@@ -161,7 +161,7 @@ The other mode, `library_variantcalling`, will call simultaneously call all libr
The config for these therefore is:
| Namespace | Name | Type | Default | Function |
| namespace | Name | Type | Default | Function |
| ----------- | ---- | ---- | ------- | -------- |
| shiva | multisample_variantcalling | Boolean | true | Default, multisample calling |
| shiva | single_sample_variantcalling | Boolean | false | Not-recommended, single sample, merged bam |
......
......@@ -26,7 +26,7 @@ class VariantRecalibrator(val root: Configurable) extends org.broadinstitute.gat
object VariantRecalibrator {
def apply(root: Configurable, input: File, recal_file: File, tranches_file: File, indel: Boolean = false): VariantRecalibrator = {
val vr = new VariantRecalibrator(root) {
override lazy val configName = "variantrecalibrator"
override lazy val configNamespace = "variantrecalibrator"
override def configPath: List[String] = (if (indel) "indel" else "snp") :: super.configPath
if (indel) {
mode = org.broadinstitute.gatk.tools.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.INDEL
......
......@@ -20,15 +20,15 @@ class Shiva(val root: Configurable) extends QScript with ShivaTrait {
qscript =>
def this() = this(null)
/** Make variantcalling submodule, this with the gatk modes in there */
/** Make variantcalling namespace, this with the gatk modes in there */
override def makeVariantcalling(multisample: Boolean = false) = {
if (multisample) new ShivaVariantcalling(qscript) {
override def namePrefix = "multisample"
override def configName = "shivavariantcalling"
override def configNamespace = "shivavariantcalling"
override def configPath: List[String] = super.configPath ::: "multisample" :: Nil
}
else new ShivaVariantcalling(qscript) {
override def configName = "shivavariantcalling"
override def configNamespace = "shivavariantcalling"
}
}
......
......@@ -27,7 +27,7 @@ import org.testng.annotations.{ DataProvider, Test }
class ShivaTest extends TestNGSuite with Matchers {
def initPipeline(map: Map[String, Any]): Shiva = {
new Shiva() {
override def configName = "shiva"
override def configNamespace = "shiva"
override def globalConfig = new Config(ConfigUtils.mergeMaps(map, ShivaTest.config))
qSettings = new QSettings
qSettings.runName = "test"
......
......@@ -29,7 +29,7 @@ import scala.collection.mutable.ListBuffer
class ShivaVariantcallingTest extends TestNGSuite with Matchers {
def initPipeline(map: Map[String, Any]): ShivaVariantcalling = {
new ShivaVariantcalling() {
override def configName = "shivavariantcalling"
override def configNamespace = "shivavariantcalling"
override def globalConfig = new Config(ConfigUtils.mergeMaps(map, ShivaVariantcallingTest.config))
qSettings = new QSettings
qSettings.runName = "test"
......
......@@ -37,7 +37,7 @@ class BamMetricsTest extends TestNGSuite with Matchers {
def initPipeline(map: Map[String, Any]): BamMetrics = {
new BamMetrics() {
override def configName = "bammetrics"
override def configNamespace = "bammetrics"
override def globalConfig = new Config(map)
qSettings = new QSettings
qSettings.runName = "test"
......
......@@ -131,11 +131,11 @@ trait BastyTrait extends MultiSampleQScript {
raxmlMl.p = Some(seed)
raxmlMl.n = outputName + "_ml"
raxmlMl.w = dirSufixRaxml
raxmlMl.N = config("ml_runs", default = 20, submodule = "raxml")
raxmlMl.N = config("ml_runs", default = 20, namespace = "raxml")
add(raxmlMl)
val r = new scala.util.Random(seed)
val numBoot = config("boot_runs", default = 100, submodule = "raxml").asInt
val numBoot = config("boot_runs", default = 100, namespace = "raxml").asInt
val bootList = for (t <- 0 until numBoot) yield {
val raxmlBoot = new Raxml(this)
raxmlBoot.input = variants
......
......@@ -30,7 +30,7 @@ import scala.collection.JavaConversions._
/** Biopet command line trait to auto check executable and cluster values */
trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
analysisName = configName
analysisName = configNamespace
@Input(doc = "deps", required = false)
var deps: List[File] = Nil
......@@ -50,9 +50,16 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
writer.println("set -eubf")
writer.println("set -o pipefail")
lines.foreach(writer.println)
jobDelayTime.foreach(x => writer.println(s"sleep $x"))
writer.close()
}
/**
* This value is used to let you job wait a x number of second after it finish.
* This is ionly used when having storage delay issues
*/
var jobDelayTime: Option[Int] = config("job_delay_time")
// This overrides the default "sh" from queue. For Biopet the default is "bash"
updateJobRun = {
case jt: JobTemplate =>
......@@ -73,6 +80,15 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
def beforeGraph() {}
override def freezeFieldValues() {
this match {
case r: Reference =>
if (r.dictRequired) deps :+= r.referenceDict
if (r.faiRequired) deps :+= r.referenceFai
deps = deps.distinct
case _ =>
}
preProcessExecutable()
beforeGraph()
internalBeforeGraph()
......@@ -165,6 +181,27 @@ trait BiopetCommandLineFunction extends CommandLineResources { biopetFunction =>
this
}
/**
* This method can handle args that have multiple args for 1 arg name
* @param argName Name of the arg like "-h" or "--help"
* @param values Values for this arg
* @param groupSize Values must come in groups of x number, default is 1
* @param minGroups Minimal groups that are required, default is 0, when 0 the method return en empty string
* @param maxGroups Max number of groups that can be filled here
* @return Command part of this arg
*/
def multiArg(argName: String, values: Iterable[Any], groupSize: Int = 1, minGroups: Int = 0, maxGroups: Int = 0): String = {
if (values.size % groupSize != 0)
Logging.addError(s"Arg '${argName}' values: '${values}' does not fit to a groupSize of ${groupSize}")
val groups = values.size / groupSize
if (groups < minGroups)
Logging.addError(s"Args '${argName}' need atleast $minGroups with size $groupSize")
if (maxGroups > 0 && groups > maxGroups)
Logging.addError(s"Args '${argName}' may only have $maxGroups with size $groupSize")
if (values.nonEmpty) required(argName) + values.map(required(_)).mkString
else ""
}
@Output(required = false)
private[core] var stdoutFile: Option[File] = None
......
......@@ -19,7 +19,7 @@ import org.broadinstitute.gatk.queue.function.JavaCommandLineFunction
/** Biopet commandline class for java based programs */
trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetCommandLineFunction {
executable = config("java", default = "java", submodule = "java", freeVar = false)
executable = config("java", default = "java", namespace = "java", freeVar = false)
javaGCThreads = config("java_gc_threads", default = 4)
javaGCHeapFreeLimit = config("java_gc_heap_freelimit", default = 10)
......
......@@ -74,7 +74,7 @@ class BiopetPipe(val commands: List[BiopetCommandLineFunction]) extends BiopetCo
override def defaultThreads = 0
val root: Configurable = commands.head.root
override def configName = commands.map(_.configName).mkString("-")
override def configNamespace = commands.map(_.configNamespace).mkString("-")
def cmdLine: String = {
"(" + commands.head.cmdLine + (if (commands.head.stdinFile.isDefined) {
" < " + required(commands.head.stdinFile.map(_.getAbsoluteFile))
......
......@@ -31,7 +31,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
@Argument(doc = "JSON / YAML config file(s)", fullName = "config_file", shortName = "config", required = false)
val configfiles: List[File] = Nil
@Argument(doc = "Config values, value should be formatted like 'key=value' or 'path:path:key=value'", fullName = "config_value", shortName = "cv", required = false)
@Argument(doc = "Config values, value should be formatted like 'key=value' or 'namespace:namespace:key=value'", fullName = "config_value", shortName = "cv", required = false)
val configValues: List[String] = Nil
/** Output directory of pipeline */
......
......@@ -88,7 +88,7 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
else residentLimit = Some((_coreMemory + (0.5 * retryMultipler)) * residentFactor)
if (!config.contains("vmem")) vmem = Some((_coreMemory * (vmemFactor + (0.5 * retryMultipler))) + "G")
jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
jobName = configNamespace + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
}
override def setupRetry(): Unit = {
......
......@@ -70,7 +70,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
def libDir = new File(sampleDir, "lib_" + libId)
lazy val libTags: Map[String, Any] =
config("tags", default = Map(), freeVar = false, submodule = libId, path = List("samples", sampleId, "libraries"))
config("tags", default = Map(), freeVar = false, namespace = libId, path = List("samples", sampleId, "libraries"))
def sampleId = sample.sampleId
......@@ -91,7 +91,7 @@ trait MultiSampleQScript extends SummaryQScript { qscript: QScript =>
val libraries: Map[String, Library] = libIds.map(id => id -> makeLibrary(id)).toMap
lazy val sampleTags: Map[String, Any] =
config("tags", default = Map(), freeVar = false, submodule = sampleId, path = List("samples"))
config("tags", default = Map(), freeVar = false, namespace = sampleId, path = List("samples"))
lazy val gender = {
val g: Option[String] = sampleTags.get("gender").map(_.toString)
......
......@@ -61,25 +61,25 @@ trait Reference extends Configurable {
}
/** When set override this on true the pipeline with raise an exception when fai index is not found */
protected def faiRequired = false
def faiRequired = false
/** When set override this on true the pipeline with raise an exception when dict index is not found */
protected def dictRequired = this.isInstanceOf[Summarizable] || this.isInstanceOf[SummaryQScript]
def dictRequired = this.isInstanceOf[Summarizable] || this.isInstanceOf[SummaryQScript]
/** Returns the dict file belonging to the fasta file */
def referenceDict = new File(referenceFasta().getAbsolutePath
.stripSuffix(".fa")
.stripSuffix(".fasta")
.stripSuffix(".fna") + ".dict")
/** Returns the fai file belonging to the fasta file */
def referenceFai = new File(referenceFasta().getAbsolutePath + ".fai")
/** Returns the fasta file */
def referenceFasta(): File = {
val file: File = config("reference_fasta")
if (config.contains("reference_fasta")) {
checkFasta(file)
val dict = new File(file.getAbsolutePath.stripSuffix(".fa").stripSuffix(".fasta").stripSuffix(".fna") + ".dict")
val fai = new File(file.getAbsolutePath + ".fai")
this match {
case c: BiopetCommandLineFunction => c.deps :::= dict :: fai :: Nil
case _ =>
}
} else {
if (config.contains("reference_fasta")) checkFasta(file)
else {
val defaults = ConfigUtils.mergeMaps(this.defaults, this.internalDefaults)
def getReferences(map: Map[String, Any]): Set[(String, String)] = (for (
......
......@@ -36,7 +36,7 @@ object WriteDependencies extends Logging with Configurable {
val cache: mutable.Map[String, Int] = mutable.Map()
for (function <- functions) {
val baseName = function match {
case f: Configurable => f.configName
case f: Configurable => f.configNamespace
case f => f.getClass.getSimpleName
}
cache += baseName -> (cache.getOrElse(baseName, 0) + 1)
......
......@@ -24,7 +24,7 @@ trait PythonCommandLineFunction extends BiopetCommandLineFunction {
@Input(doc = "Python script", required = false)
var pythonScript: File = _
executable = config("exe", default = "python", submodule = "python", freeVar = false)
executable = config("exe", default = "python", namespace = "python", freeVar = false)
protected var pythonScriptName: String = _
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment