Merge remote-tracking branch 'origin/develop' into feature-pipeable

external-example/.gitignore

+# Project-related
+dependency-reduced-pom.xml
+git.properties
+
+# gedit
+*~
+# Vim
+*.swp
+# IntelliJ
+.idea/*
+*.iml
+target/
+public/target/
+protected/target/

external-example/README.md

+Test implementation of Magpie 2.0
\ No newline at end of file

external-example/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <!--TODO: replace groupId -->
+    <groupId>org.example.group</groupId>
+
+    <!--TODO: replace artifactId -->
+    <artifactId>ExternalExample</artifactId>
+
+    <!--TODO: replace version, for a new pipeline it's advised to start with '0.1.0-SNAPSHOT' -->
+    <version>0.1.0-SNAPSHOT</version>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <scoverage.plugin.version>1.0.4</scoverage.plugin.version>
+        <sting.shade.phase>package</sting.shade.phase>
+
+        <!--
+        TODO: replace app.main.class, this is the class that get executed when running the jar file
+        This can be any executable that have a main method. In Biopet any pipeline can be used as direct executable.
+        Value for direct pipeline: 'org.example.group.pipelines.SimplePipeline'
+        In the given example is an extension of the biopet executable. In this example there are multiple pipelines in 1 executable.
+
+        It's also possible to make your own main function and call the main function with it's argument of the pipeline from there.
+        -->
+        <app.main.class>org.example.group.ExecutableExample</app.main.class>
+    </properties>
+
+    <dependencies>
+        <!--
+        In here maven dependencies can be placed, when importing a biopet pipeline 'Biopet-Framework' is not required.
+        When only using the framework without pipeline you need to import BiopetFramework.
+        It's advised to not use different versions of the pipeline and the framework.
+         -->
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>BiopetCore</artifactId>
+
+            <!--TODO: replace version of pipeline to a fixed version -->
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>BiopetExtensions</artifactId>
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>nl.lumc.sasc</groupId>
+            <artifactId>Shiva</artifactId>
+
+            <!--TODO: replace version of pipeline to a fixed version -->
+            <version>0.5.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+        <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+        <testResources>
+            <testResource>
+                <directory>${basedir}/src/test/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </testResource>
+        </testResources>
+        <resources>
+            <resource>
+                <directory>${basedir}/src/main/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.4.1</version>
+                <configuration>
+                    <!--suppress MavenModelInspection -->
+                    <finalName>${project.artifactId}-${project.version}-${git.commit.id.abbrev}</finalName>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <manifestEntries>
+                                <Main-Class>${app.main.class}</Main-Class>
+                                <!--suppress MavenModelInspection -->
+                                <X-Compile-Source-JDK>${maven.compile.source}</X-Compile-Source-JDK>
+                                <!--suppress MavenModelInspection -->
+                                <X-Compile-Target-JDK>${maven.compile.target}</X-Compile-Target-JDK>
+                            </manifestEntries>
+                        </transformer>
+                    </transformers>
+                    <filters>
+                    </filters>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.18.1</version>
+                <configuration>
+                    <forkCount>1C</forkCount>
+                    <workingDirectory>${project.build.directory}</workingDirectory>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>2.10</version>
+                <executions>
+                    <execution>
+                        <id>copy-installed</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>list</goal>
+                        </goals>
+                        <configuration>
+                            <outputFile>${project.build.outputDirectory}/dependency_list.txt</outputFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <id>scala-compile</id>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                        <configuration>
+                            <args>
+                                <arg>-dependencyfile</arg>
+                                <arg>${project.build.directory}/.scala_dependencies</arg>
+                                <arg>-deprecation</arg>
+                                <arg>-feature</arg>
+                            </args>
+                        </configuration>
+                    </execution>
+                </executions>
+                <!-- ... (see other usage or goals for details) ... -->
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.5</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+                            <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <configuration>
+                    <showDeprecation>true</showDeprecation>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.scalariform</groupId>
+                <artifactId>scalariform-maven-plugin</artifactId>
+                <version>0.1.4</version>
+                <executions>
+                    <execution>
+                        <phase>process-sources</phase>
+                        <goals>
+                            <goal>format</goal>
+                        </goals>
+                        <configuration>
+                            <rewriteArrowSymbols>false</rewriteArrowSymbols>
+                            <alignParameters>true</alignParameters>
+                            <alignSingleLineCaseStatements_maxArrowIndent>40
+                            </alignSingleLineCaseStatements_maxArrowIndent>
+                            <alignSingleLineCaseStatements>true</alignSingleLineCaseStatements>
+                            <compactStringConcatenation>false</compactStringConcatenation>
+                            <compactControlReadability>false</compactControlReadability>
+                            <doubleIndentClassDeclaration>false</doubleIndentClassDeclaration>
+                            <formatXml>true</formatXml>
+                            <indentLocalDefs>false</indentLocalDefs>
+                            <indentPackageBlocks>true</indentPackageBlocks>
+                            <indentSpaces>2</indentSpaces>
+                            <placeScaladocAsterisksBeneathSecondAsterisk>false
+                            </placeScaladocAsterisksBeneathSecondAsterisk>
+                            <preserveDanglingCloseParenthesis>true</preserveDanglingCloseParenthesis>
+                            <preserveSpaceBeforeArguments>false</preserveSpaceBeforeArguments>
+                            <rewriteArrowSymbols>false</rewriteArrowSymbols>
+                            <spaceBeforeColon>false</spaceBeforeColon>
+                            <spaceInsideBrackets>false</spaceInsideBrackets>
+                            <spaceInsideParentheses>false</spaceInsideParentheses>
+                            <spacesWithinPatternBinders>true</spacesWithinPatternBinders>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>2.1.10</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <prefix>git</prefix>
+                    <dateFormat>dd.MM.yyyy '@' HH:mm:ss z</dateFormat>
+                    <verbose>false</verbose>
+                    <!-- TODO: This directory need to be changed depening where your .git folder is relative from this pom.xml -->
+                    <dotGitDirectory>${basedir}/../.git</dotGitDirectory>
+                    <useNativeGit>true</useNativeGit>
+                    <skipPoms>false</skipPoms>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>src/main/resources/git.properties</generateGitPropertiesFilename>
+                    <failOnNoGitDirectory>false</failOnNoGitDirectory>
+                    <abbrevLength>8</abbrevLength>
+                    <skip>false</skip>
+                    <gitDescribe>
+                        <skip>false</skip>
+                        <always>false</always>
+                        <abbrev>8</abbrev>
+                        <dirty>-dirty</dirty>
+                        <forceLongFormat>false</forceLongFormat>
+                    </gitDescribe>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>com.mycila</groupId>
+                <artifactId>license-maven-plugin</artifactId>
+                <version>2.6</version>
+                <configuration>
+                    <excludes>
+                        <exclude>**/*git*</exclude>
+                        <exclude>**/License*</exclude>
+                        <exclude>**/*.bam</exclude>
+                        <exclude>**/*.bai</exclude>
+                        <exclude>**/*.gtf</exclude>
+                        <exclude>**/*.fq</exclude>
+                        <exclude>**/*.sam</exclude>
+                        <exclude>**/*.bed</exclude>
+                        <exclude>**/*.refFlat</exclude>
+                        <exclude>**/*.R</exclude>
+                        <exclude>**/*.rscript</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.scoverage</groupId>
+                <artifactId>scoverage-maven-plugin</artifactId>
+                <version>${scoverage.plugin.version}</version>
+                <configuration>
+                    <scalaVersion>2.10.2</scalaVersion>
+                    <!-- other parameters -->
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+    <reporting>
+        <plugins>
+            <plugin>
+                <groupId>org.scoverage</groupId>
+                <artifactId>scoverage-maven-plugin</artifactId>
+                <version>${scoverage.plugin.version}</version>
+            </plugin>
+        </plugins>
+    </reporting>
+</project>
\ No newline at end of file

external-example/src/main/scala/org/example/group/ExecutableExample.scala

+package org.example.group
+
+import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+object ExecutableExample extends BiopetExecutable {
+
+  /** This list defines the pipeline that are usable from the executable */
+  def pipelines: List[MainCommand] = List(
+    org.example.group.pipelines.MultisamplePipeline,
+    org.example.group.pipelines.BiopetPipeline,
+    org.example.group.pipelines.SimplePipeline
+  )
+
+  /** This list defines the (biopet)tools that are usable from the executable */
+  def tools: List[MainCommand] = Nil
+}

external-example/src/main/scala/org/example/group/pipelines/BiopetPipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.PipelineCommand
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.core.summary.SummaryQScript
+import nl.lumc.sasc.biopet.pipelines.shiva.Shiva
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvan_thof on 8/28/15.
+ */
+//TODO: Replace class Name
+class BiopetPipeline(val root: Configurable) extends QScript with SummaryQScript {
+  def this() = this(null)
+
+  /** Only required when using [[SummaryQScript]] */
+  def summaryFile = new File(outputDir, "magpie.summary.json")
+
+  /** Only required when using [[SummaryQScript]] */
+  def summaryFiles: Map[String, File] = Map()
+
+  /** Only required when using [[SummaryQScript]] */
+  def summarySettings = Map()
+
+  // This method can be used to initialize some classes where needed
+  def init(): Unit = {
+  }
+
+  // This method is the actual pipeline
+  def biopetScript: Unit = {
+
+    // Executing a biopet pipeline inside
+    val shiva = new Shiva(this)
+    shiva.init()
+    shiva.biopetScript()
+    addAll(shiva.functions)
+
+    /* Only required when using [[SummaryQScript]] */
+    addSummaryQScript(shiva)
+
+    // From here you can use the output files of shiva as input file of other jobs
+  }
+}
+
+//TODO: Replace object Name, must be the same as the class of the pipeline
+object BiopetPipeline extends PipelineCommand

external-example/src/main/scala/org/example/group/pipelines/MultisamplePipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.{ PipelineCommand, MultiSampleQScript }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+class MultisamplePipeline(val root: Configurable) extends QScript with MultiSampleQScript {
+  qscript =>
+  def this() = this(null)
+
+  def init: Unit = {
+  }
+
+  def biopetScript: Unit = {
+    addSamplesJobs() // This executes jobs for all samples
+  }
+
+  def addMultiSampleJobs: Unit = {
+    // this code will be executed after all code of all samples is executed
+  }
+
+  def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
+
+  //TODO: Add summary
+  def summaryFiles: Map[String, File] = Map()
+
+  //TODO: Add summary
+  def summarySettings: Map[String, Any] = Map()
+
+  def makeSample(id: String) = new Sample(id)
+  class Sample(sampleId: String) extends AbstractSample(sampleId) {
+
+    def makeLibrary(id: String) = new Library(id)
+    class Library(libId: String) extends AbstractLibrary(libId) {
+      //TODO: Add summary
+      def summaryFiles: Map[String, File] = Map()
+
+      //TODO: Add summary
+      def summaryStats: Map[String, Any] = Map()
+
+      def addJobs: Unit = {
+        //TODO: add library specific jobs
+      }
+    }
+
+    //TODO: Add summary
+    def summaryFiles: Map[String, File] = Map()
+
+    //TODO: Add summary
+    def summaryStats: Map[String, Any] = Map()
+
+    def addJobs: Unit = {
+      addPerLibJobs() // This add jobs for each library
+      //TODO: add sample specific jobs
+    }
+  }
+
+}
+
+object MultisamplePipeline extends PipelineCommand
\ No newline at end of file

external-example/src/main/scala/org/example/group/pipelines/SimplePipeline.scala

+package org.example.group.pipelines
+
+import nl.lumc.sasc.biopet.core.{ BiopetQScript, PipelineCommand }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import nl.lumc.sasc.biopet.extensions.{ Gzip, Cat }
+import org.broadinstitute.gatk.queue.QScript
+
+/**
+ * Created by pjvanthof on 30/08/15.
+ */
+//TODO: Replace class name, must be the same as the class of the pipeline
+class SimplePipeline(val root: Configurable) extends QScript with BiopetQScript {
+  // A constructor without arguments is needed if this pipeline is a root pipeline
+  def this() = this(null)
+
+  @Input(required = true)
+  var inputFile: File = null
+
+  /** This method can be used to initialize some classes where needed */
+  def init(): Unit = {
+  }
+
+  /** This method is the actual pipeline */
+  def biopetScript: Unit = {
+    val cat = new Cat(this)
+    cat.input :+= inputFile
+    cat.output = new File(outputDir, "file.out")
+    add(cat)
+
+    val gzip = new Gzip(this)
+    gzip.input :+= cat.output
+    gzip.output = new File(outputDir, "file.out.gz")
+    add(gzip)
+  }
+}
+
+//TODO: Replace object name, must be the same as the class of the pipeline
+object SimplePipeline extends PipelineCommand

pom.xml

@@ -16,5 +16,6 @@
     <modules>
         <module>public</module>
         <module>protected</module>
+        <module>external-example</module>
     </modules>
 </project>

protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/GatkGeneral.scala

@@ -5,7 +5,8 @@
  */
 package nl.lumc.sasc.biopet.extensions.gatk.broad
 
-import nl.lumc.sasc.biopet.core.{ BiopetJavaCommandLineFunction, Reference }
+import nl.lumc.sasc.biopet.core.{ Reference, BiopetJavaCommandLineFunction }
+import org.broadinstitute.gatk.engine.phonehome.GATKRunReport
 import org.broadinstitute.gatk.queue.extensions.gatk.CommandLineGATK
 
 trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction with Reference {
@@ -20,6 +21,15 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction wit
 
   if (config.contains("intervals")) intervals = config("intervals").asFileList
   if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
+
+  Option(config("et").value) match {
+    case Some("NO_ET")  => et = GATKRunReport.PhoneHomeOption.NO_ET
+    case Some("AWS")    => et = GATKRunReport.PhoneHomeOption.AWS
+    case Some("STDOUT") => et = GATKRunReport.PhoneHomeOption.STDOUT
+    case Some(x)        => throw new IllegalArgumentException(s"Unknown et option for gatk: $x")
+    case _              =>
+  }
+
   if (config.contains("gatk_key")) gatk_key = config("gatk_key")
   if (config.contains("pedigree")) pedigree = config("pedigree")
 

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala

@@ -93,6 +93,12 @@ class ShivaTest extends TestNGSuite with Matchers {
 
 object ShivaTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): String = {
+    val file = new File(outputDir, "input" + File.separator + name)
+    Files.touch(file)
+    file.getAbsolutePath
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)
@@ -136,8 +142,8 @@ object ShivaTest {
   val sample1 = Map(
     "samples" -> Map("sample1" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "1_1_R1.fq",
-        "R2" -> "1_1_R2.fq"
+        "R1" -> inputTouch("1_1_R1.fq"),
+        "R2" -> inputTouch("1_1_R2.fq")
       )
     )
     )))
@@ -145,8 +151,8 @@ object ShivaTest {
   val sample2 = Map(
     "samples" -> Map("sample2" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "2_1_R1.fq",
-        "R2" -> "2_1_R2.fq"
+        "R1" -> inputTouch("2_1_R1.fq"),
+        "R2" -> inputTouch("2_1_R2.fq")
       )
     )
     )))
@@ -154,12 +160,12 @@ object ShivaTest {
   val sample3 = Map(
     "samples" -> Map("sample3" -> Map("libraries" -> Map(
       "lib1" -> Map(
-        "R1" -> "3_1_R1.fq",
-        "R2" -> "3_1_R2.fq"
+        "R1" -> inputTouch("3_1_R1.fq"),
+        "R2" -> inputTouch("3_1_R2.fq")
       ),
       "lib2" -> Map(
-        "R1" -> "3_2_R1.fq",
-        "R2" -> "3_2_R2.fq"
+        "R1" -> inputTouch("3_2_R1.fq"),
+        "R2" -> inputTouch("3_2_R2.fq")
       )
     )
     )))

protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala

@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
     val map = Map("variantcallers" -> callers.toList)
     val pipeline = initPipeline(map)
 
-    pipeline.inputBams = (for (n <- 1 to bams) yield new File("bam_" + n + ".bam")).toList
+    pipeline.inputBams = (for (n <- 1 to bams) yield ShivaVariantcallingTest.inputTouch("bam_" + n + ".bam")).toList
 
     val illegalArgumentException = pipeline.inputBams.isEmpty ||
       (!raw && !bcftools &&
@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
 
 object ShivaVariantcallingTest {
   val outputDir = Files.createTempDir()
+  new File(outputDir, "input").mkdirs()
+  def inputTouch(name: String): File = {
+    val file = new File(outputDir, "input" + File.separator + name).getAbsoluteFile
+    Files.touch(file)
+    file
+  }
 
   private def copyFile(name: String): Unit = {
     val is = getClass.getResourceAsStream("/" + name)

protected/biopet-protected-package/src/main/scala/nl/lumc/sasc/biopet/BiopetExecutableProtected.scala

@@ -8,7 +8,7 @@ package nl.lumc.sasc.biopet
 import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
 
 object BiopetExecutableProtected extends BiopetExecutable {
-  def pipelines: List[MainCommand] = BiopetExecutablePublic.pipelines ::: List(