Pipeline progress, for testing the mapping pipeline

.gitignore

 dependency-reduced-pom.xml
 *~
 *.swp
-/Biopet/nbproject/private/
-/Biopet/build/
\ No newline at end of file
+test/

biopet-framework/nb-configuration.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project-shared-configuration>
+    <!--
+This file contains additional configuration written by modules in the NetBeans IDE.
+The configuration is intended to be shared among all the users of project and
+therefore it is assumed to be part of version control checkout.
+Without this configuration present, some functionality in the IDE may be limited or fail altogether.
+-->
+    <config-data xmlns="http://www.netbeans.org/ns/maven-config-data/1">
+        <configurations>
+            <configuration id="yamsvp" profiles=""/>
+        </configurations>
+    </config-data>
+    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
+        <!--
+Properties that influence various parts of the IDE, especially code formatting and the like. 
+You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
+That way multiple projects can share the same settings (useful for formatting rules for example).
+Any value defined here will override the pom.xml file value but is only applicable to the current project.
+-->
+        <netbeans.hint.license>apache20</netbeans.hint.license>
+        <com-junichi11-netbeans-changelf.enable>true</com-junichi11-netbeans-changelf.enable>
+        <com-junichi11-netbeans-changelf.use-project>true</com-junichi11-netbeans-changelf.use-project>
+        <com-junichi11-netbeans-changelf.lf-kind>CRLF</com-junichi11-netbeans-changelf.lf-kind>
+        <com-junichi11-netbeans-changelf.use-global>false</com-junichi11-netbeans-changelf.use-global>
+    </properties>
+</project-shared-configuration>

biopet-framework/nbactions-yamsvp.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>run</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath nl.lumc.sasc.biopet.pipelines.yamsvp.Yamsvp -outDir ../test -config ../test/samples.json -run</exec.args>
+                <exec.executable>java</exec.executable>
+                <exec.workingdir>../test</exec.workingdir>
+            </properties>
+        </action>
+        <action>
+            <actionName>debug</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath nl.lumc.sasc.biopet.pipelines.yamsvp.Yamsvp -outDir ../test -config ../test/samples.json -run</exec.args>
+                <exec.executable>java</exec.executable>
+                <jpda.listen>true</jpda.listen>
+                <exec.workingdir>../test</exec.workingdir>
+            </properties>
+        </action>
+        <action>
+            <actionName>profile</actionName>
+            <packagings>
+                <packaging>jar</packaging>
+            </packagings>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2.1:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath nl.lumc.sasc.biopet.pipelines.yamsvp.Yamsvp -outDir ../test -config ../test/samples.json -run</exec.args>
+                <exec.executable>java</exec.executable>
+                <exec.workingdir>../test</exec.workingdir>
+            </properties>
+        </action>
+    </actions>

biopet-framework/src/main/scala/nl/lumc/sasc/biopet/extensions/svcallers/Clever.scala

+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package nl.lumc.sasc.biopet.extensions.svcallers
+
+import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
+import nl.lumc.sasc.biopet.core.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output, Argument , MissingArgumentException}
+import java.io.File
+
+class Clever(val root: Configurable) extends BiopetCommandLineFunction  {
+  executable = config("exe", default = "clever")
+  
+  private lazy val versionexecutable: File = config("version_exe", default = (new File(executable).getParent + "/ctk-version"))
+  
+  override val defaultVmem = "4G"
+  override val defaultThreads = 8
+  
+  override def versionCommand = versionexecutable.getAbsolutePath
+  override val versionRegex = """(.*)""".r
+  
+  
+  @Input(doc = "Input file (bam)")
+  var input: File = _
+
+  @Input(doc = "Reference Fasta file")
+  var reference: File = _
+  
+  @Argument(doc = "Work directory")
+  var workdir: String = _
+  
+  @Output(doc = "Clever VCF output")
+  lazy val outputvcf: File = {
+    new File(workdir + "predictions.vcf")
+  }
+  
+  @Output(doc = "Clever raw output")
+  lazy val outputraw: File = {
+    new File(workdir + "predictions.raw.txt")
+  }
+  
+//  var vcf: File = output
+  if ( reference == null ) reference = config("reference")
+  
+  
+//  var T: Option[Int] = config("T", default = defaultThreads)
+  var f: Boolean = config("f", default = true) // delete work directory before running
+//  var w: String = config("w", default = workdir + "/work")
+  var a: Boolean = config("a", default = false) // don't recompute AS tags
+  var k: Boolean = config("k", default = false) // keep working directory
+  var r: Boolean = config("r", default = false) // take read groups into account
+  
+  override def beforeCmd {
+    if (workdir == null) throw new Exception("Clever :: Workdirectory is not defined")
+//    if (input.getName.endsWith(".sort.bam")) sorted = true
+  }
+
+  def cmdLine = required(executable) +
+      optional("-T", nCoresRequest) +
+      conditional(f ,"-f") +
+      conditional(a ,"-a") +
+      conditional(k, "-k") +
+      conditional(r ,"-r") +
+      required(this.input) +
+      required(this.reference) +
+      required(this.workdir)
+}
+
+object Clever {
+  def apply(root: Configurable, input: File, reference: File, runDir: String): Clever = {
+    val clever = new Clever(root)
+    clever.input = input
+    clever.reference = reference
+    clever.workdir = runDir
+    return clever
+  }
+}
\ No newline at end of file

biopet-framework/src/main/scala/nl/lumc/sasc/biopet/pipelines/yamsvp/Yamsvp.scala

@@ -5,22 +5,17 @@
 package nl.lumc.sasc.biopet.pipelines.yamsvp
 
 import nl.lumc.sasc.biopet.core.config.Configurable
-import nl.lumc.sasc.biopet.core.BiopetQScript
 import nl.lumc.sasc.biopet.core.MultiSampleQScript
 import nl.lumc.sasc.biopet.core.PipelineCommand
 
 import nl.lumc.sasc.biopet.extensions.Cat
 import nl.lumc.sasc.biopet.extensions.picard.MergeSamFiles
+import nl.lumc.sasc.biopet.extensions.svcallers.Clever
 
-import nl.lumc.sasc.biopet.pipelines.flexiprep.Flexiprep
 import nl.lumc.sasc.biopet.pipelines.mapping.Mapping
 
-import nl.lumc.sasc.biopet.scripts.PrefixFastq
-
 import org.broadinstitute.gatk.queue.QScript
 import org.broadinstitute.gatk.queue.function._
-import org.broadinstitute.gatk.utils.commandline.{ Argument }
-
 
 
 
@@ -30,9 +25,6 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
   var reference: File = _
   var finalBamFiles: List[File] = Nil
   
-  @Input(doc = "countBed", required = false)
-  var countBed : File = _
-  
   def init() {
     for (file <- configfiles) globalConfig.loadConfigFile(file)
     reference = config("reference", required = true)
@@ -47,8 +39,9 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
     // start with QC, alignment, call sambamba, call sv callers, reporting
 
     // read config and set all parameters for the pipeline
-    logger.warn("runSamplesJobs")
+    logger.info("Starting YAM SV Pipeline")
     runSamplesJobs
+    // 
 
   }
   
@@ -59,7 +52,6 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
     val sampleID: String = sampleConfig("ID").toString
     val sampleDir: String = outputDir + sampleID + "/" 
     for ((library, libraryFiles) <- runLibraryJobs(sampleConfig)) {
-      libraryFastqFiles +:= libraryFiles("prefix_fastq")
       libraryBamfiles +:= libraryFiles("FinalBam")
     }
     
@@ -75,7 +67,13 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
                     add(cat)
                     cat.output
                   } else null
-    
+
+    /// bamfile will be used as input for the SV callers. First run Clever
+//    val cleverVCF : File = sampleDir + "/" + sampleID + ".clever.vcf"
+//    
+//    val clever = Clever(this, bamFile, this.reference, sampleDir )
+//    outputFiles += ("CleverVCF" -> List(clever.outputvcf) )
+//    add(clever)
     return outputFiles
   }
   
@@ -86,29 +84,12 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
     val sampleID: String = sampleConfig("ID").toString
     val runDir: String = outputDir + sampleID + "/run_" + runID + "/"
     if (runConfig.contains("R1")) {
-      val flexiprep = new Flexiprep(this)
-      flexiprep.outputDir = runDir + "flexiprep/"
-      flexiprep.input_R1 = new File(runConfig("R1").toString)
-      flexiprep.skipClip = true
-      flexiprep.skipTrim = true
-      flexiprep.sampleName = sampleID
-      flexiprep.libraryName = runID
-      flexiprep.init
-      flexiprep.biopetScript
-      addAll(flexiprep.functions)
-      
-      val flexiprepOutput = for ((key,file) <- flexiprep.outputFiles if key.endsWith("output_R1")) yield file
-      val prefixFastq = PrefixFastq.apply(this, flexiprepOutput.head, runDir)
-      prefixFastq.prefix = config("sage_tag", default = "CATG")
-      prefixFastq.deps +:= flexiprep.outputFiles("fastq_input_R1")
-      add(prefixFastq)
-      outputFiles += ("prefix_fastq" -> prefixFastq.output)
-      
-      val mapping = new Mapping(this)
-      mapping.skipFlexiprep = true
+      val mapping = Mapping.loadFromLibraryConfig(this, runConfig, sampleConfig, runDir)
+      mapping.skipFlexiprep = false
       mapping.skipMarkduplicates = true
       mapping.defaultAligner = "bwa"
-      mapping.input_R1 = prefixFastq.output
+      mapping.chunking = true
+      mapping.numberChunks = 4
       mapping.RGLB = runConfig("ID").toString
       mapping.RGSM = sampleConfig("ID").toString
       if (runConfig.contains("PL")) mapping.RGPL = runConfig("PL").toString
@@ -118,12 +99,12 @@ class Yamsvp(val root: Configurable) extends QScript with MultiSampleQScript {
       mapping.init
       mapping.biopetScript
       addAll(mapping.functions)
-      
+
       outputFiles += ("FinalBam" -> mapping.outputFiles("finalBamFile"))
     } else this.logger.error("Sample: " + sampleID + ": No R1 found for run: " + runConfig)
+    logger.debug( outputFiles )
     return outputFiles
   }
-  
 }
 
 object Yamsvp extends PipelineCommand {