Commit 6fc77a78 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Fixed resources

parent 3ae49381
......@@ -60,7 +60,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
// This overrides the default "sh" from queue. For Biopet the default is "bash"
updateJobRun = {
case jt: JobTemplate => jt.setRemoteCommand(remoteCommand)
case jt: JobTemplate => jt.setRemoteCommand(remoteCommand)
case ps: ProcessSettings => ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail)
}
......@@ -78,6 +78,8 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
beforeGraph()
internalBeforeGraph()
if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
super.freezeFieldValues()
}
......@@ -95,10 +97,12 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
if (jobOutputFile == null && firstOutput != null)
jobOutputFile = new File(firstOutput.getAbsoluteFile.getParent, "." + firstOutput.getName + "." + configName + ".out")
if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).sum
if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).map(i => if (i == 0) 1 else i).sum
if (threads > 1) nCoresRequest = Option(threads)
_coreMemory = config("core_memory", default = defaultCoreMemory + pipesJobs.map(_.coreMemeory).sum).asDouble + (0.5 * retry)
_coreMemory = config("core_memory", default = defaultCoreMemory +
pipesJobs.map(job => job.coreMemeory * (job.threads.toDouble / this.threads.toDouble)).sum).asDouble +
(0.5 * retry)
if (config.contains("memory_limit")) memoryLimit = config("memory_limit")
else memoryLimit = Some(_coreMemory * threads)
......@@ -107,7 +111,6 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
else residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor)
if (!config.contains("vmem")) vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry))) + "G")
if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
}
......@@ -317,6 +320,10 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
}
private[core] var pipesJobs: List[BiopetCommandLineFunction] = Nil
def addPipeJob(job: BiopetCommandLineFunction) {
pipesJobs :+= job
pipesJobs = pipesJobs.distinct
}
def requiredInput(prefix: String, arg: Either[File, BiopetCommandLineFunction]): String = {
arg match {
......@@ -326,7 +333,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
}
case Right(cmd) => {
cmd._outputAsStdout = true
pipesJobs :+= cmd
addPipeJob(cmd)
try {
if (cmd.outputs != null) outputFiles ++= cmd.outputs
if (cmd.inputs != null) deps ++= cmd.inputs
......@@ -346,7 +353,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
}
case Right(cmd) => {
cmd._inputAsStdin = true
pipesJobs :+= cmd
addPipeJob(cmd)
try {
if (cmd.outputs != null) outputFiles ++= cmd.outputs
if (cmd.inputs != null) deps ++= cmd.inputs
......
......@@ -38,7 +38,7 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
if (javaMainClass != null && javaClasspath.isEmpty)
javaClasspath = JavaCommandLineFunction.currentClasspath
threads = getThreads(defaultThreads)
//threads = getThreads(defaultThreads)
}
/** Creates command to execute extension */
......
......@@ -82,6 +82,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
case f: BiopetCommandLineFunction =>
f.preProcessExecutable()
f.beforeGraph()
f.internalBeforeGraph()
f.commandLine
case _ =>
}
......
......@@ -52,7 +52,18 @@ class FastqSync(val root: Configurable) extends ToolCommandFuntion with Summariz
override def defaultCoreMemory = 4.0
// executed command line
override def beforeGraph(): Unit = {
super.beforeGraph()
inputFastq1 match {
case Right(job) => addPipeJob(job)
case _ =>
}
inputFastq2 match {
case Right(job) => addPipeJob(job)
case _ =>
}
}
override def cmdLine =
super.cmdLine +
required("-r", refFastq) +
......
......@@ -199,7 +199,6 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
fqSync.inputFastq2 = Right(qcCmdR2)
fqSync.outputFastq1 = fastqR1Qc
fqSync.outputFastq2 = fastqR2Qc.get
fqSync.threads = 6
fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats")
fqSync.isIntermediate = !keepQcFastqFiles
fqSync.deps ::= fastqc_R1.output
......
......@@ -28,17 +28,21 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
var read: String = _
override def defaultCoreMemory = 2.0
override def defaultThreads = 3
val seqtk = new SeqtkSeq(root)
var clip: Option[Cutadapt] = None
var trim: Option[Sickle] = None
override def beforeGraph(): Unit = {
super.beforeGraph()
require(read != null)
deps ::= input
outputFiles :+= output
}
override def defaultCoreMemory = 2.0
override def defaultThreads = 3
def cmdLine = {
val seqtk = new SeqtkSeq(root)
override def beforeCmd(): Unit = {
seqtk.input = input
seqtk.Q = fastqc.encoding match {
case null => None
......@@ -50,7 +54,7 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
}
if (seqtk.Q.isDefined) seqtk.V = true
val clip = if (!flexiprep.skipClip) {
clip = if (!flexiprep.skipClip) {
val foundAdapters = fastqc.foundAdapters.map(_.seq)
if (foundAdapters.nonEmpty) {
val cutadept = new nl.lumc.sasc.biopet.extensions.Cutadapt(root)
......@@ -62,11 +66,14 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
} else None
} else None
val trim = if (!flexiprep.skipTrim) {
trim = if (!flexiprep.skipTrim) {
val sickle = new nl.lumc.sasc.biopet.extensions.Sickle(root)
sickle.output_stats = new File(flexiprep.outputDir, s"${flexiprep.sampleId.getOrElse("x")}-${flexiprep.libId.getOrElse("x")}.$read.trim.stats")
Some(sickle)
} else None
}
def cmdLine = {
val outputCommand = {
if (compress) new Gzip(root)
else new Cat(root)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment