Merge branch 'develop' into feature-fixed_values

external-example/.gitignore

+# Project-related
+dependency-reduced-pom.xml
+git.properties
+
+# gedit
+*~
+# Vim
+*.swp
+# IntelliJ
+.idea/*
+*.iml
+target/
+public/target/
+protected/target/

external-example/README.md

+Test implementation of Magpie 2.0
\ No newline at end of file

external-example/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <!--TODO: replace groupId -->
+    <groupId>org.example.group</groupId>
+
+    <!--TODO: replace artifactId -->
+    <artifactId>ExternalExample</artifactId>
+
+    <!--TODO: replace version, for a new pipeline it's advised to start with '0.1.0-SNAPSHOT' -->
+    <version>0.1.0-SNAPSHOT</version>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <scoverage.plugin.version>1.0.4</scoverage.plugin.version>
+        <sting.shade.phase>package</sting.shade.phase>
+
+        <!--
+        TODO: replace app.main.class, this is the class that get executed when running the jar file
+        This can be any executable that have a main method. In Biopet any pipeline can be used as direct executable.
+        Value for direct pipeline: 'org.example.group.pipelines.SimplePipeline'
+        In the given example is an extension of the biopet executable. In this example there are multiple pipelines in 1 executable.
+
+        It's also possible to make your own main function and call the main function with it's argument of the pipeline from there.
+        -->
+        <app.main.class>org.example.group.ExecutableExample</app.main.class>
+    </properties>
+
+    <dependencies>
+        <!--
+        In here maven dependencies can be placed, when importing a biopet pipeline 'Biopet-Framework' is not required.
+        When only using the framework without pipeline you need to import BiopetFramework.
+        It's advised to not use different versions of the pipeline and the framework.
+         -->
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>BiopetCore</artifactId>
+
+            <!--TODO: replace version of pipeline to a fixed version -->
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>BiopetExtensions</artifactId>
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>Shiva</artifactId>
+
+            <!--TODO: replace version of pipeline to a fixed version -->
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+        <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+        <testResources>
+            <testResource>
+                <directory>${basedir}/src/test/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </testResource>
+        </testResources>
+        <resources>
+            <resource>
+                <directory>${basedir}/src/main/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.4.1</version>
+                <configuration>
+                    <!--suppress MavenModelInspection -->
+                    <finalName>${project.artifactId}-${project.version}-${git.commit.id.abbrev}</finalName>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <manifestEntries>
+                                <Main-Class>${app.main.class}</Main-Class>
+                                <!--suppress MavenModelInspection -->
+                                <X-Compile-Source-JDK>${maven.compile.source}</X-Compile-Source-JDK>
+                                <!--suppress MavenModelInspection -->
+                                <X-Compile-Target-JDK>${maven.compile.target}</X-Compile-Target-JDK>
+                            </manifestEntries>
+                        </transformer>
+                    </transformers>
+                    <filters>
+                    </filters>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.18.1</version>
+                <configuration>
+                    <forkCount>1C</forkCount>
+                    <workingDirectory>${project.build.directory}</workingDirectory>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>2.10</version>
+                <executions>
+                    <execution>
+                        <id>copy-installed</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>list</goal>
+                        </goals>
+                        <configuration>
+                            <outputFile>${project.build.outputDirectory}/dependency_list.txt</outputFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <id>scala-compile</id>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                        <configuration>
+                            <args>
+                                <arg>-dependencyfile</arg>
+                                <arg>${project.build.directory}/.scala_dependencies</arg>
+                                <arg>-deprecation</arg>
+                                <arg>-feature</arg>
+                            </args>
+                        </configuration>
+                    </execution>
+                </executions>
+                <!-- ... (see other usage or goals for details) ... -->
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.5</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+                            <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <configuration>
+                    <showDeprecation>true</showDeprecation>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.scalariform</groupId>
+                <artifactId>scalariform-maven-plugin</artifactId>
+                <version>0.1.4</version>
+                <executions>
+                    <execution>
+                        <phase>process-sources</phase>
+                        <goals>
+                            <goal>format</goal>
+                        </goals>
+                        <configuration>
+                            <rewriteArrowSymbols>false</rewriteArrowSymbols>
+                            <alignParameters>true</alignParameters>
+                            <alignSingleLineCaseStatements_maxArrowIndent>40
+                            </alignSingleLineCaseStatements_maxArrowIndent>
+                            <alignSingleLineCaseStatements>true</alignSingleLineCaseStatements>
+                            <compactStringConcatenation>false</compactStringConcatenation>
+                            <compactControlReadability>false</compactControlReadability>
+                            <doubleIndentClassDeclaration>false</doubleIndentClassDeclaration>
+                            <formatXml>true</formatXml>
+                            <indentLocalDefs>false</indentLocalDefs>
+                            <indentPackageBlocks>true</indentPackageBlocks>
+                            <indentSpaces>2</indentSpaces>
+                            <placeScaladocAsterisksBeneathSecondAsterisk>false
+                            </placeScaladocAsterisksBeneathSecondAsterisk>
+                            <preserveDanglingCloseParenthesis>true</preserveDanglingCloseParenthesis>
+                            <preserveSpaceBeforeArguments>false</preserveSpaceBeforeArguments>
+                            <rewriteArrowSymbols>false</rewriteArrowSymbols>
+                            <spaceBeforeColon>false</spaceBeforeColon>
+                            <spaceInsideBrackets>false</spaceInsideBrackets>
+                            <spaceInsideParentheses>false</spaceInsideParentheses>
+                            <spacesWithinPatternBinders>true</spacesWithinPatternBinders>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>2.1.10</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <prefix>git</prefix>
+                    <dateFormat>dd.MM.yyyy '@' HH:mm:ss z</dateFormat>
+                    <verbose>false</verbose>
+                    <!-- TODO: This directory need to be changed depening where your .git folder is relative from this pom.xml -->
+                    <dotGitDirectory>${basedir}/../.git</dotGitDirectory>
+                    <useNativeGit>true</useNativeGit>
+                    <skipPoms>false</skipPoms>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>src/main/resources/git.properties</generateGitPropertiesFilename>
+                    <failOnNoGitDirectory>false</failOnNoGitDirectory>
+                    <abbrevLength>8</abbrevLength>
+                    <skip>false</skip>
+                    <gitDescribe>
+                        <skip>false</skip>
+                        <always>false</always>
+                        <abbrev>8</abbrev>
+                        <dirty>-dirty</dirty>
+                        <forceLongFormat>false</forceLongFormat>
+                    </gitDescribe>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>com.mycila</groupId>
+                <artifactId>license-maven-plugin</artifactId>
+                <version>2.6</version>
+                <configuration>
+                    <excludes>
+                        <exclude>**/*git*</exclude>
+                        <exclude>**/License*</exclude>
+                        <exclude>**/*.bam</exclude>
+                        <exclude>**/*.bai</exclude>
+                        <exclude>**/*.gtf</exclude>
+                        <exclude>**/*.fq</exclude>
+                        <exclude>**/*.sam</exclude>
+                        <exclude>**/*.bed</exclude>
+                        <exclude>**/*.refFlat</exclude>
+                        <exclude>**/*.R</exclude>
+                        <exclude>**/*.rscript</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.scoverage</groupId>
+                <artifactId>scoverage-maven-plugin</artifactId>
+                <version>${scoverage.plugin.version}</version>
+                <configuration>
+                    <scalaVersion>2.10.2</scalaVersion>
+                    <!-- other parameters -->
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.scoverage</groupId>
+                <artifactId>scoverage-maven-plugin</artifactId>
+                <version>${scoverage.plugin.version}</version>
+            </plugin>
+        </plugins>
+    </reporting>
+</project>
\ No newline at end of file

external-example/src/main/scala/org/example/group/ExecutableExample.scala

+package org.example.group
+
+import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+object ExecutableExample extends BiopetExecutable {
+
+  /** This list defines the pipeline that are usable from the executable */
+  def pipelines: List[MainCommand] = List(
+    org.example.group.pipelines.MultisamplePipeline,
+    org.example.group.pipelines.BiopetPipeline,
+    org.example.group.pipelines.SimplePipeline
+  )
+
+  /** This list defines the (biopet)tools that are usable from the executable */
+  def tools: List[MainCommand] = Nil
+}

external-example/src/main/scala/org/example/group/pipelines/BiopetPipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.PipelineCommand
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.core.summary.SummaryQScript
+import nl.lumc.sasc.biopet.pipelines.shiva.Shiva
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvan_thof on 8/28/15.
+ */
+//TODO: Replace class Name
+class BiopetPipeline(val root: Configurable) extends QScript with SummaryQScript {
+  def this() = this(null)
+
+  /** Only required when using [[SummaryQScript]] */
+  def summaryFile = new File(outputDir, "magpie.summary.json")
+
+  /** Only required when using [[SummaryQScript]] */
+  def summaryFiles: Map[String, File] = Map()
+
+  /** Only required when using [[SummaryQScript]] */
+  def summarySettings = Map()
+
+  // This method can be used to initialize some classes where needed
+  def init(): Unit = {
+  }
+
+  // This method is the actual pipeline
+  def biopetScript: Unit = {
+
+    // Executing a biopet pipeline inside
+    val shiva = new Shiva(this)
+    shiva.init()
+    shiva.biopetScript()
+    addAll(shiva.functions)
+
+    /* Only required when using [[SummaryQScript]] */
+    addSummaryQScript(shiva)
+
+    // From here you can use the output files of shiva as input file of other jobs
+  }
+}
+
+//TODO: Replace object Name, must be the same as the class of the pipeline
+object BiopetPipeline extends PipelineCommand

external-example/src/main/scala/org/example/group/pipelines/MultisamplePipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.{ PipelineCommand, MultiSampleQScript }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+class MultisamplePipeline(val root: Configurable) extends QScript with MultiSampleQScript {
+  qscript =>
+  def this() = this(null)
+
+  def init: Unit = {
+  }
+
+  def biopetScript: Unit = {
+    addSamplesJobs() // This executes jobs for all samples
+  }
+
+  def addMultiSampleJobs: Unit = {
+    // this code will be executed after all code of all samples is executed
+  }
+
+  def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
+
+  //TODO: Add summary
+  def summaryFiles: Map[String, File] = Map()
+
+  //TODO: Add summary
+  def summarySettings: Map[String, Any] = Map()
+
+  def makeSample(id: String) = new Sample(id)
+  class Sample(sampleId: String) extends AbstractSample(sampleId) {
+
+    def makeLibrary(id: String) = new Library(id)
+    class Library(libId: String) extends AbstractLibrary(libId) {
+      //TODO: Add summary
+      def summaryFiles: Map[String, File] = Map()
+
+      //TODO: Add summary
+      def summaryStats: Map[String, Any] = Map()
+
+      def addJobs: Unit = {
+        //TODO: add library specific jobs
+      }
+    }
+
+    //TODO: Add summary
+    def summaryFiles: Map[String, File] = Map()
+
+    //TODO: Add summary
+    def summaryStats: Map[String, Any] = Map()
+
+    def addJobs: Unit = {
+      addPerLibJobs() // This add jobs for each library
+      //TODO: add sample specific jobs
+    }
+  }
+
+}
+
+object MultisamplePipeline extends PipelineCommand
\ No newline at end of file

external-example/src/main/scala/org/example/group/pipelines/SimplePipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.{ BiopetQScript, PipelineCommand }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.extensions.{ Gzip, Cat }
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+//TODO: Replace class name, must be the same as the class of the pipeline
+class SimplePipeline(val root: Configurable) extends QScript with BiopetQScript {
+  // A constructor without arguments is needed if this pipeline is a root pipeline
+  def this() = this(null)
+
+  @Input(required = true)
+  var inputFile: File = null
+
+  /** This method can be used to initialize some classes where needed */
+  def init(): Unit = {
+  }
+
+  /** This method is the actual pipeline */
+  def biopetScript: Unit = {
+    val cat = new Cat(this)
+    cat.input :+= inputFile
+    cat.output = new File(outputDir, "file.out")
+    add(cat)
+
+    val gzip = new Gzip(this)
+    gzip.input :+= cat.output
+    gzip.output = new File(outputDir, "file.out.gz")
+    add(gzip)
+  }
+}
+
+//TODO: Replace object name, must be the same as the class of the pipeline
+object SimplePipeline extends PipelineCommand

pom.xml

@@ -16,5 +16,6 @@
     <modules>
         <module>public</module>
         <module>protected</module>
+        <module>external-example</module>
     </modules>
 </project>

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/ApplyRecalibration.scala

@@ -14,8 +14,8 @@ class ApplyRecalibration(val root: Configurable) extends org.broadinstitute.gatk
 
   override val defaultThreads = 3
 
-  override def beforeGraph() {
-    super.beforeGraph()
+  override def freezeFieldValues() {
+    super.freezeFieldValues()
 
     nt = Option(getThreads)
     memoryLimit = Option(nt.getOrElse(1) * 2)

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/BaseRecalibrator.scala

@@ -20,7 +20,6 @@ object BaseRecalibrator {
     val br = new BaseRecalibrator(root)
     br.input_file :+= input
     br.out = output
-    br.beforeGraph()
     br
   }
 }
\ No newline at end of file

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/GatkGeneral.scala

@@ -5,32 +5,39 @@
  */
 package nl.lumc.sasc.biopet.extensions.gatk.broad
 
-import nl.lumc.sasc.biopet.core.{ BiopetJavaCommandLineFunction, Reference }
+import nl.lumc.sasc.biopet.core.{ CommandLineResources, Reference, BiopetJavaCommandLineFunction }
+import org.broadinstitute.gatk.engine.phonehome.GATKRunReport
 import org.broadinstitute.gatk.queue.extensions.gatk.CommandLineGATK
 
-trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction with Reference {
+trait GatkGeneral extends CommandLineGATK with CommandLineResources with Reference {
   memoryLimit = Option(3)
 
   override def subPath = "gatk" :: super.subPath
 
   jarFile = config("gatk_jar")
 
+  reference_sequence = referenceFasta()
+
   override def defaultCoreMemory = 4.0
   override def faiRequired = true
 
   if (config.contains("intervals")) intervals = config("intervals").asFileList
   if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
+
+  Option(config("et").value) match {
+    case Some("NO_ET")  => et = GATKRunReport.PhoneHomeOption.NO_ET
+    case Some("AWS")    => et = GATKRunReport.PhoneHomeOption.AWS
+    case Some("STDOUT") => et = GATKRunReport.PhoneHomeOption.STDOUT
+    case Some(x)        => throw new IllegalArgumentException(s"Unknown et option for gatk: $x")
+    case _              =>
+  }
+
   if (config.contains("gatk_key")) gatk_key = config("gatk_key")
   if (config.contains("pedigree")) pedigree = config("pedigree")
 
-  override def versionRegex = """(.*)""".r
-  override def versionExitcode = List(0, 1)
-  override def versionCommand = executable + " -jar " + jarFile + " -version"
-
-  override def getVersion = super.getVersion.collect { case v => "Gatk " + v }
+  //override def versionRegex = """(.*)""".r
+  //override def versionExitcode = List(0, 1)
+  //override def versionCommand = executable + " -jar " + jarFile + " -version"
 
-  override def beforeGraph(): Unit = {
-    super.beforeGraph()
-    if (reference_sequence == null) reference_sequence = referenceFasta()
-  }
+  //override def getVersion = super.getVersion.collect { case v => "Gatk " + v }
 }

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/HaplotypeCaller.scala

@@ -38,11 +38,11 @@ class HaplotypeCaller(val root: Configurable) extends org.broadinstitute.gatk.qu
     stand_emit_conf = config("stand_emit_conf", default = 0)
   }
 
-  override def beforeGraph() {
-    super.beforeGraph()
+  override def freezeFieldValues() {
+    super.freezeFieldValues()
     if (bamOutput != null && nct.getOrElse(1) > 1) {
-      threads = 1
       logger.warn("BamOutput is on, nct/threads is forced to set on 1, this option is only for debug")
+      nCoresRequest = Some(1)
     }
     nct = Some(getThreads)
     memoryLimit = Option(memoryLimit.getOrElse(2.0) * nct.getOrElse(1))

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/UnifiedGenotyper.scala

@@ -28,8 +28,8 @@ class UnifiedGenotyper(val root: Configurable) extends org.broadinstitute.gatk.q
 
   override val defaultThreads = 1
 
-  override def beforeGraph() {
-    super.beforeGraph()
+  override def freezeFieldValues() {
+    super.freezeFieldValues()
 
     genotype_likelihoods_model = org.broadinstitute.gatk.tools.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.BOTH
     nct = Some(getThreads)

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/VariantEval.scala

@@ -10,9 +10,6 @@ import java.io.File
 import nl.lumc.sasc.biopet.utils.config.Configurable