Merge branch 'feature-auto_memory' into 'develop'

Feature auto memory

Implementation for #136, not yet tested but this should work I think.

See merge request !139

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/BaseRecalibrator.scala

@@ -9,9 +9,6 @@ import java.io.File
 import nl.lumc.sasc.biopet.core.config.Configurable
 
 class BaseRecalibrator(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.BaseRecalibrator with GatkGeneral {
-  memoryLimit = Option(4)
-  override val defaultVmem = "8G"
-
   if (config.contains("scattercount")) scatterCount = config("scattercount", default = 1)
   if (config.contains("dbsnp")) knownSites :+= new File(config("dbsnp").asString)
   if (config.contains("known_sites")) knownSites :+= new File(config("known_sites").asString)

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/GatkGeneral.scala

@@ -15,7 +15,7 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction {
 
   jarFile = config("gatk_jar")
 
-  override val defaultVmem = "7G"
+  override val defaultCoreMemory = 4.0
 
   if (config.contains("intervals")) intervals = config("intervals").asFileList
   if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/PrintReads.scala

@@ -9,10 +9,6 @@ import java.io.File
 import nl.lumc.sasc.biopet.core.config.Configurable
 
 class PrintReads(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.PrintReads with GatkGeneral {
-  memoryLimit = Option(4)
-
-  override val defaultVmem = "8G"
-
   if (config.contains("scattercount")) scatterCount = config("scattercount")
 }
 

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/RealignerTargetCreator.scala

@@ -9,9 +9,6 @@ import java.io.File
 import nl.lumc.sasc.biopet.core.config.Configurable
 
 class RealignerTargetCreator(val root: Configurable) extends org.broadinstitute.gatk.queue.extensions.gatk.RealignerTargetCreator with GatkGeneral {
-  override val defaultVmem = "6G"
-  memoryLimit = Some(2.5)
-
   if (config.contains("scattercount")) scatterCount = config("scattercount")
 
   if (config.contains("known")) known ++= config("known").asFileList

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetCommandLineFunctionTrait.scala

@@ -35,8 +35,14 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
   var threads = 0
   val defaultThreads = 1
 
-  var vmem: Option[String] = None
-  val defaultVmem: String = ""
+  var vmem: Option[String] = config("vmem")
+  protected val defaultCoreMemory: Double = 1.0
+  var vmemFactor: Double = config("vmem_factor", default = 1.5)
+
+  var residentFactor: Double = config("resident_factor", default = 1.2)
+
+  private var coreMemory: Double = _
+
   var executable: String = _
 
   /**
@@ -57,16 +63,31 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
     if (threads == 0) threads = getThreads(defaultThreads)
     if (threads > 1) nCoresRequest = Option(threads)
 
-    if (vmem.isEmpty) {
-      vmem = config("vmem")
-      if (vmem.isEmpty && defaultVmem.nonEmpty) vmem = Some(defaultVmem)
-    }
+    coreMemory = config("core_memory", default = defaultCoreMemory).asDouble + (0.5 * retry)
+
+    if (config.contains("memory_limit")) memoryLimit = config("memory_limit")
+    else memoryLimit = Some(coreMemory * threads)
+
+    if (config.contains("resident_limit")) residentLimit = config("resident_limit")
+    else residentLimit = Some((coreMemory + (0.5 * retry)) * residentFactor)
+
+    if (!config.contains("vmem")) vmem = Some((coreMemory * (vmemFactor + (0.5 * retry))) + "G")
     if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
     jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
 
     super.freezeFieldValues()
   }
 
+  var retry = 0
+
+  override def setupRetry(): Unit = {
+    super.setupRetry()
+    if (vmem.isDefined) jobResourceRequests = jobResourceRequests.filterNot(_.contains("h_vmem="))
+    logger.info("Auto raise memory on retry")
+    retry += 1
+    this.freeze()
+  }
+
   /** can override this value is executable may not be converted to CanonicalPath */
   val executableToCanonicalPath = true
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetJavaCommandLineFunction.scala

@@ -29,12 +29,6 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
   override def javaOpts = super.javaOpts +
     optional("-Dscala.concurrent.context.numThreads=", threads, spaceSeparated = false, escape = false)
 
-  /** Sets memory limit */
-  override def beforeGraph {
-    super.beforeGraph
-    if (memoryLimit.isEmpty) memoryLimit = config("memory_limit")
-  }
-
   /** Creates command to execute extension */
   override def commandLine: String = {
     preCmdInternal

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Bowtie.scala

@@ -44,7 +44,7 @@ class Bowtie(val root: Configurable) extends BiopetCommandLineFunction {
   override val versionExitcode = List(0, 1)
   override def versionCommand = executable + " --version"
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   var sam: Boolean = config("sam", default = true)

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Cufflinks.scala

@@ -33,7 +33,7 @@ class Cufflinks(val root: Configurable) extends BiopetCommandLineFunction {
   override val defaultThreads = 8
 
   /** default vmem for cluster jobs */
-  override val defaultVmem = "8G"
+  override val defaultCoreMemory = 6.0
 
   /** input file */
   @Input(doc = "Input file (SAM or BAM)", required = true)

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Gsnap.scala

@@ -35,7 +35,7 @@ class Gsnap(val root: Configurable) extends BiopetCommandLineFunction {
   override val defaultThreads = 8
 
   /** default vmem for cluster jobs */
-  override val defaultVmem = "16G"
+  override val defaultCoreMemory = 10.0
 
   /** input file */
   @Input(doc = "Input FASTQ file(s)", required = true) //var input: List[File] = _

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Pbzip2.scala

@@ -33,7 +33,7 @@ class Pbzip2(val root: Configurable) extends BiopetCommandLineFunction {
   var decomrpess = true
   var memory: Option[Int] = config("memory")
 
-  override val defaultVmem = (memory.getOrElse(1000) * 2 / 1000) + "G"
+  override val defaultCoreMemory = memory.getOrElse(1000).toDouble / 1000
   override val defaultThreads = 2
 
   override def beforeCmd {

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/RscriptCommandLineFunction.scala

@@ -36,7 +36,7 @@ trait RscriptCommandLineFunction extends BiopetCommandLineFunction {
   protected var script: File = _
   protected var scriptName: String = _
   protected var arguments: ListBuffer[String] = ListBuffer()
-  override val defaultVmem: String = "4G"
+  override val defaultCoreMemory = 2.0
 
   executable = config("exe", default = "Rscript", submodule = "R")
 

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Stampy.scala

@@ -64,7 +64,7 @@ class Stampy(val root: Configurable) extends BiopetCommandLineFunction {
   override val versionExitcode = List(0, 1)
 
   /// Stampy uses approx factor 1.1 times the size of the genome in memory.
-  override val defaultVmem = "4G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   override def versionCommand = executable + " --help"

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Star.scala

@@ -63,7 +63,7 @@ class Star(val root: Configurable) extends BiopetCommandLineFunction {
   var outFileNamePrefix: String = _
   var runThreadN: Option[Int] = config("runThreadN")
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   /** Sets output files for the graph */

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/Tophat.scala

@@ -32,7 +32,7 @@ class Tophat(val root: Configurable) extends BiopetCommandLineFunction {
   override val versionExitcode = List(0, 1)
   override def versionCommand = executable + " --version"
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   @Input(doc = "FastQ file(s) R1", shortName = "R1")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/bedtools/BedtoolsCoverage.scala

@@ -46,7 +46,7 @@ class BedtoolsCoverage(val root: Configurable) extends Bedtools {
     if (input.getName.endsWith(".bam")) inputTag = "-abam"
   }
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
 
   /** Returns command to execute */
   def cmdLine = required(executable) + required("coverage") +

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/bwa/BwaAln.scala

@@ -60,7 +60,7 @@ class BwaAln(val root: Configurable) extends Bwa {
   var n2: Boolean = config("2", default = false)
   var Y: Boolean = config("Y", default = false)
 
-  override val defaultVmem = "5G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   /** Returns command to execute */

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/bwa/BwaMem.scala

@@ -69,7 +69,7 @@ class BwaMem(val root: Configurable) extends Bwa {
   var Y: Boolean = config("Y", default = false)
   var I: Option[String] = config("I")
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
   override val defaultThreads = 8
 
   def cmdLine = {

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/conifer/Conifer.scala

@@ -26,7 +26,7 @@ abstract class Conifer extends PythonCommandLineFunction {
   override val versionExitcode = List(0)
   override def versionCommand = executable + " " + python_script + " --version"
 
-  override val defaultVmem = "8G"
+  override val defaultCoreMemory = 5.0
   override val defaultThreads = 1
 
   def cmdLine = getPythonCommand

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/Gatk.scala

@@ -31,7 +31,7 @@ abstract class Gatk extends BiopetJavaCommandLineFunction {
 
   val analysisType: String
 
-  override val defaultVmem = "5G"
+  override val defaultCoreMemory = 3.0
 
   @Input(required = true)
   var reference: File = config("reference")

public/biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/igvtools/IGVToolsCount.scala

@@ -59,7 +59,7 @@ class IGVToolsCount(val root: Configurable) extends IGVTools {
 
   var pairs: Boolean = config("pairs", default = false)
 
-  override val defaultVmem = "6G"
+  override val defaultCoreMemory = 4.0
 
   override def beforeGraph {
     super.beforeGraph