Commit 6fc77a78 authored by Peter van 't Hof's avatar Peter van 't Hof

Fixed resources

parent 3ae49381
...@@ -60,7 +60,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -60,7 +60,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
// This overrides the default "sh" from queue. For Biopet the default is "bash" // This overrides the default "sh" from queue. For Biopet the default is "bash"
updateJobRun = { updateJobRun = {
case jt: JobTemplate => jt.setRemoteCommand(remoteCommand) case jt: JobTemplate => jt.setRemoteCommand(remoteCommand)
case ps: ProcessSettings => ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail) case ps: ProcessSettings => ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail)
} }
...@@ -78,6 +78,8 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -78,6 +78,8 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
beforeGraph() beforeGraph()
internalBeforeGraph() internalBeforeGraph()
if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
super.freezeFieldValues() super.freezeFieldValues()
} }
...@@ -95,10 +97,12 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -95,10 +97,12 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
if (jobOutputFile == null && firstOutput != null) if (jobOutputFile == null && firstOutput != null)
jobOutputFile = new File(firstOutput.getAbsoluteFile.getParent, "." + firstOutput.getName + "." + configName + ".out") jobOutputFile = new File(firstOutput.getAbsoluteFile.getParent, "." + firstOutput.getName + "." + configName + ".out")
if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).sum if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).map(i => if (i == 0) 1 else i).sum
if (threads > 1) nCoresRequest = Option(threads) if (threads > 1) nCoresRequest = Option(threads)
_coreMemory = config("core_memory", default = defaultCoreMemory + pipesJobs.map(_.coreMemeory).sum).asDouble + (0.5 * retry) _coreMemory = config("core_memory", default = defaultCoreMemory +
pipesJobs.map(job => job.coreMemeory * (job.threads.toDouble / this.threads.toDouble)).sum).asDouble +
(0.5 * retry)
if (config.contains("memory_limit")) memoryLimit = config("memory_limit") if (config.contains("memory_limit")) memoryLimit = config("memory_limit")
else memoryLimit = Some(_coreMemory * threads) else memoryLimit = Some(_coreMemory * threads)
...@@ -107,7 +111,6 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -107,7 +111,6 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
else residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor) else residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor)
if (!config.contains("vmem")) vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry))) + "G") if (!config.contains("vmem")) vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry))) + "G")
if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile) jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
} }
...@@ -317,6 +320,10 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -317,6 +320,10 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
} }
private[core] var pipesJobs: List[BiopetCommandLineFunction] = Nil private[core] var pipesJobs: List[BiopetCommandLineFunction] = Nil
def addPipeJob(job: BiopetCommandLineFunction) {
pipesJobs :+= job
pipesJobs = pipesJobs.distinct
}
def requiredInput(prefix: String, arg: Either[File, BiopetCommandLineFunction]): String = { def requiredInput(prefix: String, arg: Either[File, BiopetCommandLineFunction]): String = {
arg match { arg match {
...@@ -326,7 +333,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -326,7 +333,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
} }
case Right(cmd) => { case Right(cmd) => {
cmd._outputAsStdout = true cmd._outputAsStdout = true
pipesJobs :+= cmd addPipeJob(cmd)
try { try {
if (cmd.outputs != null) outputFiles ++= cmd.outputs if (cmd.outputs != null) outputFiles ++= cmd.outputs
if (cmd.inputs != null) deps ++= cmd.inputs if (cmd.inputs != null) deps ++= cmd.inputs
...@@ -346,7 +353,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable { ...@@ -346,7 +353,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
} }
case Right(cmd) => { case Right(cmd) => {
cmd._inputAsStdin = true cmd._inputAsStdin = true
pipesJobs :+= cmd addPipeJob(cmd)
try { try {
if (cmd.outputs != null) outputFiles ++= cmd.outputs if (cmd.outputs != null) outputFiles ++= cmd.outputs
if (cmd.inputs != null) deps ++= cmd.inputs if (cmd.inputs != null) deps ++= cmd.inputs
......
...@@ -38,7 +38,7 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC ...@@ -38,7 +38,7 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
if (javaMainClass != null && javaClasspath.isEmpty) if (javaMainClass != null && javaClasspath.isEmpty)
javaClasspath = JavaCommandLineFunction.currentClasspath javaClasspath = JavaCommandLineFunction.currentClasspath
threads = getThreads(defaultThreads) //threads = getThreads(defaultThreads)
} }
/** Creates command to execute extension */ /** Creates command to execute extension */
......
...@@ -82,6 +82,7 @@ trait BiopetQScript extends Configurable with GatkLogging { ...@@ -82,6 +82,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
case f: BiopetCommandLineFunction => case f: BiopetCommandLineFunction =>
f.preProcessExecutable() f.preProcessExecutable()
f.beforeGraph() f.beforeGraph()
f.internalBeforeGraph()
f.commandLine f.commandLine
case _ => case _ =>
} }
......
...@@ -52,7 +52,18 @@ class FastqSync(val root: Configurable) extends ToolCommandFuntion with Summariz ...@@ -52,7 +52,18 @@ class FastqSync(val root: Configurable) extends ToolCommandFuntion with Summariz
override def defaultCoreMemory = 4.0 override def defaultCoreMemory = 4.0
// executed command line override def beforeGraph(): Unit = {
super.beforeGraph()
inputFastq1 match {
case Right(job) => addPipeJob(job)
case _ =>
}
inputFastq2 match {
case Right(job) => addPipeJob(job)
case _ =>
}
}
override def cmdLine = override def cmdLine =
super.cmdLine + super.cmdLine +
required("-r", refFastq) + required("-r", refFastq) +
......
...@@ -199,7 +199,6 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with ...@@ -199,7 +199,6 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
fqSync.inputFastq2 = Right(qcCmdR2) fqSync.inputFastq2 = Right(qcCmdR2)
fqSync.outputFastq1 = fastqR1Qc fqSync.outputFastq1 = fastqR1Qc
fqSync.outputFastq2 = fastqR2Qc.get fqSync.outputFastq2 = fastqR2Qc.get
fqSync.threads = 6
fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats") fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats")
fqSync.isIntermediate = !keepQcFastqFiles fqSync.isIntermediate = !keepQcFastqFiles
fqSync.deps ::= fastqc_R1.output fqSync.deps ::= fastqc_R1.output
......
...@@ -28,17 +28,21 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman ...@@ -28,17 +28,21 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
var read: String = _ var read: String = _
override def defaultCoreMemory = 2.0
override def defaultThreads = 3
val seqtk = new SeqtkSeq(root)
var clip: Option[Cutadapt] = None
var trim: Option[Sickle] = None
override def beforeGraph(): Unit = { override def beforeGraph(): Unit = {
super.beforeGraph() super.beforeGraph()
require(read != null) require(read != null)
deps ::= input deps ::= input
outputFiles :+= output
} }
override def defaultCoreMemory = 2.0 override def beforeCmd(): Unit = {
override def defaultThreads = 3
def cmdLine = {
val seqtk = new SeqtkSeq(root)
seqtk.input = input seqtk.input = input
seqtk.Q = fastqc.encoding match { seqtk.Q = fastqc.encoding match {
case null => None case null => None
...@@ -50,7 +54,7 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman ...@@ -50,7 +54,7 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
} }
if (seqtk.Q.isDefined) seqtk.V = true if (seqtk.Q.isDefined) seqtk.V = true
val clip = if (!flexiprep.skipClip) { clip = if (!flexiprep.skipClip) {
val foundAdapters = fastqc.foundAdapters.map(_.seq) val foundAdapters = fastqc.foundAdapters.map(_.seq)
if (foundAdapters.nonEmpty) { if (foundAdapters.nonEmpty) {
val cutadept = new nl.lumc.sasc.biopet.extensions.Cutadapt(root) val cutadept = new nl.lumc.sasc.biopet.extensions.Cutadapt(root)
...@@ -62,11 +66,14 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman ...@@ -62,11 +66,14 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
} else None } else None
} else None } else None
val trim = if (!flexiprep.skipTrim) { trim = if (!flexiprep.skipTrim) {
val sickle = new nl.lumc.sasc.biopet.extensions.Sickle(root) val sickle = new nl.lumc.sasc.biopet.extensions.Sickle(root)
sickle.output_stats = new File(flexiprep.outputDir, s"${flexiprep.sampleId.getOrElse("x")}-${flexiprep.libId.getOrElse("x")}.$read.trim.stats") sickle.output_stats = new File(flexiprep.outputDir, s"${flexiprep.sampleId.getOrElse("x")}-${flexiprep.libId.getOrElse("x")}.$read.trim.stats")
Some(sickle) Some(sickle)
} else None } else None
}
def cmdLine = {
val outputCommand = { val outputCommand = {
if (compress) new Gzip(root) if (compress) new Gzip(root)
else new Cat(root) else new Cat(root)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment