Commit 04f35d97 authored by Wai Yi Leung's avatar Wai Yi Leung
Browse files

Merge branch 'feature-auto_memory' into 'develop'

Feature auto memory

Implementation for #136, not yet tested but this should work I think.

See merge request !139
parents 7fb6a028 c47680a6
......@@ -9,9 +9,6 @@ import java.io.File
import nl.lumc.sasc.biopet.core.config.Configurable
class BaseRecalibrator(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.BaseRecalibrator with GatkGeneral {
memoryLimit = Option(4)
override val defaultVmem = "8G"
if (config.contains("scattercount")) scatterCount = config("scattercount", default = 1)
if (config.contains("dbsnp")) knownSites :+= new File(config("dbsnp").asString)
if (config.contains("known_sites")) knownSites :+= new File(config("known_sites").asString)
......
......@@ -15,7 +15,7 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction {
jarFile = config("gatk_jar")
override val defaultVmem = "7G"
override val defaultCoreMemory = 4.0
if (config.contains("intervals")) intervals = config("intervals").asFileList
if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
......
......@@ -9,10 +9,6 @@ import java.io.File
import nl.lumc.sasc.biopet.core.config.Configurable
class PrintReads(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.PrintReads with GatkGeneral {
memoryLimit = Option(4)
override val defaultVmem = "8G"
if (config.contains("scattercount")) scatterCount = config("scattercount")
}
......
......@@ -9,9 +9,6 @@ import java.io.File
import nl.lumc.sasc.biopet.core.config.Configurable
class RealignerTargetCreator(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.RealignerTargetCreator with GatkGeneral {
override val defaultVmem = "6G"
memoryLimit = Some(2.5)
if (config.contains("scattercount")) scatterCount = config("scattercount")
if (config.contains("known")) known ++= config("known").asFileList
......
......@@ -35,8 +35,14 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
var threads = 0
val defaultThreads = 1
var vmem: Option[String] = None
val defaultVmem: String = ""
var vmem: Option[String] = config("vmem")
protected val defaultCoreMemory: Double = 1.0
var vmemFactor: Double = config("vmem_factor", default = 1.5)
var residentFactor: Double = config("resident_factor", default = 1.2)
private var coreMemory: Double = _
var executable: String = _
/**
......@@ -57,16 +63,31 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
if (threads == 0) threads = getThreads(defaultThreads)
if (threads > 1) nCoresRequest = Option(threads)
if (vmem.isEmpty) {
vmem = config("vmem")
if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
}
coreMemory = config("core_memory", default = defaultCoreMemory).asDouble + (0.5 * retry)
if (config.contains("memory_limit")) memoryLimit = config("memory_limit")
else memoryLimit = Some(coreMemory * threads)
if (config.contains("resident_limit")) residentLimit = config("resident_limit")
else residentLimit = Some((coreMemory + (0.5 * retry)) * residentFactor)
if (!config.contains("vmem")) vmem = Some((coreMemory * (vmemFactor + (0.5 * retry))) + "G")
if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
super.freezeFieldValues()
}
var retry = 0
override def setupRetry(): Unit = {
super.setupRetry()
if (vmem.isDefined) jobResourceRequests = jobResourceRequests.filterNot(_.contains("h_vmem="))
logger.info("Auto raise memory on retry")
retry += 1
this.freeze()
}
/** can override this value is executable may not be converted to CanonicalPath */
val executableToCanonicalPath = true
......
......@@ -29,12 +29,6 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
override def javaOpts = super.javaOpts +
optional("-Dscala.concurrent.context.numThreads=", threads, spaceSeparated = false, escape = false)
/** Sets memory limit */
override def beforeGraph {
super.beforeGraph
if (memoryLimit.isEmpty) memoryLimit = config("memory_limit")
}
/** Creates command to execute extension */
override def commandLine: String = {
preCmdInternal
......
......@@ -44,7 +44,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction {
override val versionExitcode = List(0, 1)
override def versionCommand = executable + " --version"
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
var sam: Boolean = config("sam", default = true)
......
......@@ -33,7 +33,7 @@ class Cufflinks(val root: Configurable) extends BiopetCommandLineFunction {
override val defaultThreads = 8
/** default vmem for cluster jobs */
override val defaultVmem = "8G"
override val defaultCoreMemory = 6.0
/** input file */
@Input(doc = "Input file (SAM or BAM)", required = true)
......
......@@ -35,7 +35,7 @@ class Gsnap(val root: Configurable) extends BiopetCommandLineFunction {
override val defaultThreads = 8
/** default vmem for cluster jobs */
override val defaultVmem = "16G"
override val defaultCoreMemory = 10.0
/** input file */
@Input(doc = "Input FASTQ file(s)", required = true) //var input: List[File] = _
......
......@@ -33,7 +33,7 @@ class Pbzip2(val root: Configurable) extends BiopetCommandLineFunction {
var decomrpess = true
var memory: Option[Int] = config("memory")
override val defaultVmem = (memory.getOrElse(1000) * 2 / 1000) + "G"
override val defaultCoreMemory = memory.getOrElse(1000).toDouble / 1000
override val defaultThreads = 2
override def beforeCmd {
......
......@@ -36,7 +36,7 @@ trait RscriptCommandLineFunction extends BiopetCommandLineFunction {
protected var script: File = _
protected var scriptName: String = _
protected var arguments: ListBuffer[String] = ListBuffer()
override val defaultVmem: String = "4G"
override val defaultCoreMemory = 2.0
executable = config("exe", default = "Rscript", submodule = "R")
......
......@@ -64,7 +64,7 @@ class Stampy(val root: Configurable) extends BiopetCommandLineFunction {
override val versionExitcode = List(0, 1)
/// Stampy uses approx factor 1.1 times the size of the genome in memory.
override val defaultVmem = "4G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
override def versionCommand = executable + " --help"
......
......@@ -63,7 +63,7 @@ class Star(val root: Configurable) extends BiopetCommandLineFunction {
var outFileNamePrefix: String = _
var runThreadN: Option[Int] = config("runThreadN")
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
/** Sets output files for the graph */
......
......@@ -32,7 +32,7 @@ class Tophat(val root: Configurable) extends BiopetCommandLineFunction {
override val versionExitcode = List(0, 1)
override def versionCommand = executable + " --version"
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
@Input(doc = "FastQ file(s) R1", shortName = "R1")
......
......@@ -46,7 +46,7 @@ class BedtoolsCoverage(val root: Configurable) extends Bedtools {
if (input.getName.endsWith(".bam")) inputTag = "-abam"
}
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
/** Returns command to execute */
def cmdLine = required(executable) + required("coverage") +
......
......@@ -60,7 +60,7 @@ class BwaAln(val root: Configurable) extends Bwa {
var n2: Boolean = config("2", default = false)
var Y: Boolean = config("Y", default = false)
override val defaultVmem = "5G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
/** Returns command to execute */
......
......@@ -69,7 +69,7 @@ class BwaMem(val root: Configurable) extends Bwa {
var Y: Boolean = config("Y", default = false)
var I: Option[String] = config("I")
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
override val defaultThreads = 8
def cmdLine = {
......
......@@ -26,7 +26,7 @@ abstract class Conifer extends PythonCommandLineFunction {
override val versionExitcode = List(0)
override def versionCommand = executable + " " + python_script + " --version"
override val defaultVmem = "8G"
override val defaultCoreMemory = 5.0
override val defaultThreads = 1
def cmdLine = getPythonCommand
......
......@@ -31,7 +31,7 @@ abstract class Gatk extends BiopetJavaCommandLineFunction {
val analysisType: String
override val defaultVmem = "5G"
override val defaultCoreMemory = 3.0
@Input(required = true)
var reference: File = config("reference")
......
......@@ -59,7 +59,7 @@ class IGVToolsCount(val root: Configurable) extends IGVTools {
var pairs: Boolean = config("pairs", default = false)
override val defaultVmem = "6G"
override val defaultCoreMemory = 4.0
override def beforeGraph {
super.beforeGraph
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment