Fixed resources

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetCommandLineFunction.scala

@@ -60,7 +60,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
 
   // This overrides the default "sh" from queue. For Biopet the default is "bash"
   updateJobRun = {
-    case jt: JobTemplate => jt.setRemoteCommand(remoteCommand)
+    case jt: JobTemplate     => jt.setRemoteCommand(remoteCommand)
     case ps: ProcessSettings => ps.setCommand(Array(remoteCommand) ++ ps.getCommand.tail)
   }
 
@@ -78,6 +78,8 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
     beforeGraph()
     internalBeforeGraph()
 
+    if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
+
     super.freezeFieldValues()
   }
 
@@ -95,10 +97,12 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
     if (jobOutputFile == null && firstOutput != null)
       jobOutputFile = new File(firstOutput.getAbsoluteFile.getParent, "." + firstOutput.getName + "." + configName + ".out")
 
-    if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).sum
+    if (threads == 0) threads = getThreads(defaultThreads) + pipesJobs.map(_.threads).map(i => if (i == 0) 1 else i).sum
     if (threads > 1) nCoresRequest = Option(threads)
 
-    _coreMemory = config("core_memory", default = defaultCoreMemory + pipesJobs.map(_.coreMemeory).sum).asDouble + (0.5 * retry)
+    _coreMemory = config("core_memory", default = defaultCoreMemory +
+      pipesJobs.map(job => job.coreMemeory * (job.threads.toDouble / this.threads.toDouble)).sum).asDouble +
+      (0.5 * retry)
 
     if (config.contains("memory_limit")) memoryLimit = config("memory_limit")
     else memoryLimit = Some(_coreMemory * threads)
@@ -107,7 +111,6 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
     else residentLimit = Some((_coreMemory + (0.5 * retry)) * residentFactor)
 
     if (!config.contains("vmem")) vmem = Some((_coreMemory * (vmemFactor + (0.5 * retry))) + "G")
-    if (vmem.isDefined) jobResourceRequests :+= "h_vmem=" + vmem.get
     jobName = configName + ":" + (if (firstOutput != null) firstOutput.getName else jobOutputFile)
   }
 
@@ -317,6 +320,10 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
   }
 
   private[core] var pipesJobs: List[BiopetCommandLineFunction] = Nil
+  def addPipeJob(job: BiopetCommandLineFunction) {
+    pipesJobs :+= job
+    pipesJobs = pipesJobs.distinct
+  }
 
   def requiredInput(prefix: String, arg: Either[File, BiopetCommandLineFunction]): String = {
     arg match {
@@ -326,7 +333,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
       }
       case Right(cmd) => {
         cmd._outputAsStdout = true
-        pipesJobs :+= cmd
+        addPipeJob(cmd)
         try {
           if (cmd.outputs != null) outputFiles ++= cmd.outputs
           if (cmd.inputs != null) deps ++= cmd.inputs
@@ -346,7 +353,7 @@ trait BiopetCommandLineFunction extends CommandLineFunction with Configurable {
       }
       case Right(cmd) => {
         cmd._inputAsStdin = true
-        pipesJobs :+= cmd
+        addPipeJob(cmd)
         try {
           if (cmd.outputs != null) outputFiles ++= cmd.outputs
           if (cmd.inputs != null) deps ++= cmd.inputs

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetJavaCommandLineFunction.scala

@@ -38,7 +38,7 @@ trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetC
     if (javaMainClass != null && javaClasspath.isEmpty)
       javaClasspath = JavaCommandLineFunction.currentClasspath
 
-    threads = getThreads(defaultThreads)
+    //threads = getThreads(defaultThreads)
   }
 
   /** Creates command to execute extension */

public/biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -82,6 +82,7 @@ trait BiopetQScript extends Configurable with GatkLogging {
       case f: BiopetCommandLineFunction =>
         f.preProcessExecutable()
         f.beforeGraph()
+        f.internalBeforeGraph()
         f.commandLine
       case _ =>
     }

public/biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/FastqSync.scala

@@ -52,7 +52,18 @@ class FastqSync(val root: Configurable) extends ToolCommandFuntion with Summariz
 
   override def defaultCoreMemory = 4.0
 
-  // executed command line
+  override def beforeGraph(): Unit = {
+    super.beforeGraph()
+    inputFastq1 match {
+      case Right(job) => addPipeJob(job)
+      case _          =>
+    }
+    inputFastq2 match {
+      case Right(job) => addPipeJob(job)
+      case _          =>
+    }
+  }
+
   override def cmdLine =
     super.cmdLine +
       required("-r", refFastq) +

public/flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala

@@ -199,7 +199,6 @@ class Flexiprep(val root: Configurable) extends QScript with SummaryQScript with
       fqSync.inputFastq2 = Right(qcCmdR2)
       fqSync.outputFastq1 = fastqR1Qc
       fqSync.outputFastq2 = fastqR2Qc.get
-      fqSync.threads = 6
       fqSync.outputStats = new File(outDir, s"${sampleId.getOrElse("x")}-${libId.getOrElse("x")}.sync.stats")
       fqSync.isIntermediate = !keepQcFastqFiles
       fqSync.deps ::= fastqc_R1.output

public/flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/QcCommand.scala

@@ -28,17 +28,21 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
 
   var read: String = _
 
+  override def defaultCoreMemory = 2.0
+  override def defaultThreads = 3
+
+  val seqtk = new SeqtkSeq(root)
+  var clip: Option[Cutadapt] = None
+  var trim: Option[Sickle] = None
+
   override def beforeGraph(): Unit = {
     super.beforeGraph()
     require(read != null)
     deps ::= input
+    outputFiles :+= output
   }
 
-  override def defaultCoreMemory = 2.0
-  override def defaultThreads = 3
-
-  def cmdLine = {
-    val seqtk = new SeqtkSeq(root)
+  override def beforeCmd(): Unit = {
     seqtk.input = input
     seqtk.Q = fastqc.encoding match {
       case null => None
@@ -50,7 +54,7 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
     }
     if (seqtk.Q.isDefined) seqtk.V = true
 
-    val clip = if (!flexiprep.skipClip) {
+    clip = if (!flexiprep.skipClip) {
       val foundAdapters = fastqc.foundAdapters.map(_.seq)
       if (foundAdapters.nonEmpty) {
         val cutadept = new nl.lumc.sasc.biopet.extensions.Cutadapt(root)
@@ -62,11 +66,14 @@ class QcCommand(val root: Configurable, val fastqc: Fastqc) extends BiopetComman
       } else None
     } else None
 
-    val trim = if (!flexiprep.skipTrim) {
+    trim = if (!flexiprep.skipTrim) {
       val sickle = new nl.lumc.sasc.biopet.extensions.Sickle(root)
       sickle.output_stats = new File(flexiprep.outputDir, s"${flexiprep.sampleId.getOrElse("x")}-${flexiprep.libId.getOrElse("x")}.$read.trim.stats")
       Some(sickle)
     } else None
+  }
+
+  def cmdLine = {
     val outputCommand = {
       if (compress) new Gzip(root)
       else new Cat(root)