Skip to content

GitLab

Commit 1dd927a3 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Merge remote-tracking branch 'origin/develop' into feature-pipeable

parents f7ea04a7 f4063de9
Hide whitespace changes
Inline Side-by-side
Showing with 560 additions and 26 deletions
external-example/.gitignore 0 → 100644
View file @ 1dd927a3
# Project-related
dependency-reduced-pom.xml
git.properties
# gedit
*~
# Vim
*.swp
# IntelliJ
.idea/*
*.iml
target/
public/target/
protected/target/
external-example/README.md 0 → 100644
View file @ 1dd927a3
Test implementation of Magpie 2.0
\ No newline at end of file
external-example/pom.xml 0 → 100644
View file @ 1dd927a3
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<!--TODO: replace groupId -->
<groupId>org.example.group</groupId>
<!--TODO: replace artifactId -->
<artifactId>ExternalExample</artifactId>
<!--TODO: replace version, for a new pipeline it's advised to start with '0.1.0-SNAPSHOT' -->
<version>0.1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<scoverage.plugin.version>1.0.4</scoverage.plugin.version>
<sting.shade.phase>package</sting.shade.phase>
<!--
TODO: replace app.main.class, this is the class that get executed when running the jar file
This can be any executable that have a main method. In Biopet any pipeline can be used as direct executable.
Value for direct pipeline: 'org.example.group.pipelines.SimplePipeline'
In the given example is an extension of the biopet executable. In this example there are multiple pipelines in 1 executable.
It's also possible to make your own main function and call the main function with it's argument of the pipeline from there.
-->
<app.main.class>org.example.group.ExecutableExample</app.main.class>
</properties>
<dependencies>
<!--
In here maven dependencies can be placed, when importing a biopet pipeline 'Biopet-Framework' is not required.
When only using the framework without pipeline you need to import BiopetFramework.
It's advised to not use different versions of the pipeline and the framework.
-->
<dependency>
<groupId>nl.lumc.sasc</groupId>
<artifactId>BiopetCore</artifactId>
<!--TODO: replace version of pipeline to a fixed version -->
<version>0.5.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>nl.lumc.sasc</groupId>
<artifactId>BiopetExtensions</artifactId>
<version>0.5.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>nl.lumc.sasc</groupId>
<artifactId>Shiva</artifactId>
<!--TODO: replace version of pipeline to a fixed version -->
<version>0.5.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
<testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
<testResources>
<testResource>
<directory>${basedir}/src/test/resources</directory>
<includes>
<include>**/*</include>
</includes>
</testResource>
</testResources>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>**/*</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!--suppress MavenModelInspection -->
<finalName>${project.artifactId}-${project.version}-${git.commit.id.abbrev}</finalName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Main-Class>${app.main.class}</Main-Class>
<!--suppress MavenModelInspection -->
<X-Compile-Source-JDK>${maven.compile.source}</X-Compile-Source-JDK>
<!--suppress MavenModelInspection -->
<X-Compile-Target-JDK>${maven.compile.target}</X-Compile-Target-JDK>
</manifestEntries>
</transformer>
</transformers>
<filters>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
<configuration>
<forkCount>1C</forkCount>
<workingDirectory>${project.build.directory}</workingDirectory>
</configuration>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy-installed</id>
<phase>prepare-package</phase>
<goals>
<goal>list</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/dependency_list.txt</outputFile>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<args>
<arg>-dependencyfile</arg>
<arg>${project.build.directory}/.scala_dependencies</arg>
<arg>-deprecation</arg>
<arg>-feature</arg>
</args>
</configuration>
</execution>
</executions>
<!-- ... (see other usage or goals for details) ... -->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<showDeprecation>true</showDeprecation>
</configuration>
</plugin>
<plugin>
<groupId>org.scalariform</groupId>
<artifactId>scalariform-maven-plugin</artifactId>
<version>0.1.4</version>
<executions>
<execution>
<phase>process-sources</phase>
<goals>
<goal>format</goal>
</goals>
<configuration>
<rewriteArrowSymbols>false</rewriteArrowSymbols>
<alignParameters>true</alignParameters>
<alignSingleLineCaseStatements_maxArrowIndent>40
</alignSingleLineCaseStatements_maxArrowIndent>
<alignSingleLineCaseStatements>true</alignSingleLineCaseStatements>
<compactStringConcatenation>false</compactStringConcatenation>
<compactControlReadability>false</compactControlReadability>
<doubleIndentClassDeclaration>false</doubleIndentClassDeclaration>
<formatXml>true</formatXml>
<indentLocalDefs>false</indentLocalDefs>
<indentPackageBlocks>true</indentPackageBlocks>
<indentSpaces>2</indentSpaces>
<placeScaladocAsterisksBeneathSecondAsterisk>false
</placeScaladocAsterisksBeneathSecondAsterisk>
<preserveDanglingCloseParenthesis>true</preserveDanglingCloseParenthesis>
<preserveSpaceBeforeArguments>false</preserveSpaceBeforeArguments>
<rewriteArrowSymbols>false</rewriteArrowSymbols>
<spaceBeforeColon>false</spaceBeforeColon>
<spaceInsideBrackets>false</spaceInsideBrackets>
<spaceInsideParentheses>false</spaceInsideParentheses>
<spacesWithinPatternBinders>true</spacesWithinPatternBinders>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>2.1.10</version>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<prefix>git</prefix>
<dateFormat>dd.MM.yyyy '@' HH:mm:ss z</dateFormat>
<verbose>false</verbose>
<!-- TODO: This directory need to be changed depening where your .git folder is relative from this pom.xml -->
<dotGitDirectory>${basedir}/../.git</dotGitDirectory>
<useNativeGit>true</useNativeGit>
<skipPoms>false</skipPoms>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<generateGitPropertiesFilename>src/main/resources/git.properties</generateGitPropertiesFilename>
<failOnNoGitDirectory>false</failOnNoGitDirectory>
<abbrevLength>8</abbrevLength>
<skip>false</skip>
<gitDescribe>
<skip>false</skip>
<always>false</always>
<abbrev>8</abbrev>
<dirty>-dirty</dirty>
<forceLongFormat>false</forceLongFormat>
</gitDescribe>
</configuration>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>2.6</version>
<configuration>
<excludes>
<exclude>**/*git*</exclude>
<exclude>**/License*</exclude>
<exclude>**/*.bam</exclude>
<exclude>**/*.bai</exclude>
<exclude>**/*.gtf</exclude>
<exclude>**/*.fq</exclude>
<exclude>**/*.sam</exclude>
<exclude>**/*.bed</exclude>
<exclude>**/*.refFlat</exclude>
<exclude>**/*.R</exclude>
<exclude>**/*.rscript</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>${scoverage.plugin.version}</version>
<configuration>
<scalaVersion>2.10.2</scalaVersion>
<!-- other parameters -->
</configuration>
</plugin>
</plugins>
</build>
<reporting>
<plugins>
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>${scoverage.plugin.version}</version>
</plugin>
</plugins>
</reporting>
</project>
\ No newline at end of file
external-example/src/main/scala/org/example/group/ExecutableExample.scala 0 → 100644
View file @ 1dd927a3
package org.example.group
import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
/**
* Created by pjvanthof on 30/08/15.
*/
object ExecutableExample extends BiopetExecutable {
/** This list defines the pipeline that are usable from the executable */
def pipelines: List[MainCommand] = List(
org.example.group.pipelines.MultisamplePipeline,
org.example.group.pipelines.BiopetPipeline,
org.example.group.pipelines.SimplePipeline
)
/** This list defines the (biopet)tools that are usable from the executable */
def tools: List[MainCommand] = Nil
}
external-example/src/main/scala/org/example/group/pipelines/BiopetPipeline.scala 0 → 100644
View file @ 1dd927a3
package org.example.group.pipelines
import nl.lumc.sasc.biopet.core.PipelineCommand
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.core.summary.SummaryQScript
import nl.lumc.sasc.biopet.pipelines.shiva.Shiva
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvan_thof on 8/28/15.
*/
//TODO: Replace class Name
class BiopetPipeline(val root: Configurable) extends QScript with SummaryQScript {
def this() = this(null)
/** Only required when using [[SummaryQScript]] */
def summaryFile = new File(outputDir, "magpie.summary.json")
/** Only required when using [[SummaryQScript]] */
def summaryFiles: Map[String, File] = Map()
/** Only required when using [[SummaryQScript]] */
def summarySettings = Map()
// This method can be used to initialize some classes where needed
def init(): Unit = {
}
// This method is the actual pipeline
def biopetScript: Unit = {
// Executing a biopet pipeline inside
val shiva = new Shiva(this)
shiva.init()
shiva.biopetScript()
addAll(shiva.functions)
/* Only required when using [[SummaryQScript]] */
addSummaryQScript(shiva)
// From here you can use the output files of shiva as input file of other jobs
}
}
//TODO: Replace object Name, must be the same as the class of the pipeline
object BiopetPipeline extends PipelineCommand
external-example/src/main/scala/org/example/group/pipelines/MultisamplePipeline.scala 0 → 100644
View file @ 1dd927a3
package org.example.group.pipelines
import nl.lumc.sasc.biopet.core.{ PipelineCommand, MultiSampleQScript }
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvanthof on 30/08/15.
*/
class MultisamplePipeline(val root: Configurable) extends QScript with MultiSampleQScript {
qscript =>
def this() = this(null)
def init: Unit = {
}
def biopetScript: Unit = {
addSamplesJobs() // This executes jobs for all samples
}
def addMultiSampleJobs: Unit = {
// this code will be executed after all code of all samples is executed
}
def summaryFile: File = new File(outputDir, "MultisamplePipeline.summary.json")
//TODO: Add summary
def summaryFiles: Map[String, File] = Map()
//TODO: Add summary
def summarySettings: Map[String, Any] = Map()
def makeSample(id: String) = new Sample(id)
class Sample(sampleId: String) extends AbstractSample(sampleId) {
def makeLibrary(id: String) = new Library(id)
class Library(libId: String) extends AbstractLibrary(libId) {
//TODO: Add summary
def summaryFiles: Map[String, File] = Map()
//TODO: Add summary
def summaryStats: Map[String, Any] = Map()
def addJobs: Unit = {
//TODO: add library specific jobs
}
}
//TODO: Add summary
def summaryFiles: Map[String, File] = Map()
//TODO: Add summary
def summaryStats: Map[String, Any] = Map()
def addJobs: Unit = {
addPerLibJobs() // This add jobs for each library
//TODO: add sample specific jobs
}
}
}
object MultisamplePipeline extends PipelineCommand
\ No newline at end of file
external-example/src/main/scala/org/example/group/pipelines/SimplePipeline.scala 0 → 100644
View file @ 1dd927a3
package org.example.group.pipelines
import nl.lumc.sasc.biopet.core.{ BiopetQScript, PipelineCommand }
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.extensions.{ Gzip, Cat }
import org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvanthof on 30/08/15.
*/
//TODO: Replace class name, must be the same as the class of the pipeline
class SimplePipeline(val root: Configurable) extends QScript with BiopetQScript {
// A constructor without arguments is needed if this pipeline is a root pipeline
def this() = this(null)
@Input(required = true)
var inputFile: File = null
/** This method can be used to initialize some classes where needed */
def init(): Unit = {
}
/** This method is the actual pipeline */
def biopetScript: Unit = {
val cat = new Cat(this)
cat.input :+= inputFile
cat.output = new File(outputDir, "file.out")
add(cat)
val gzip = new Gzip(this)
gzip.input :+= cat.output
gzip.output = new File(outputDir, "file.out.gz")
add(gzip)
}
}
//TODO: Replace object name, must be the same as the class of the pipeline
object SimplePipeline extends PipelineCommand
pom.xml
View file @ 1dd927a3
......@@ -16,5 +16,6 @@
<modules>
<module>public</module>
<module>protected</module>
<module>external-example</module>
</modules>
</project>
protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/GatkGeneral.scala
View file @ 1dd927a3
......@@ -5,7 +5,8 @@
*/
package nl.lumc.sasc.biopet.extensions.gatk.broad
import nl.lumc.sasc.biopet.core.{ BiopetJavaCommandLineFunction, Reference }
import nl.lumc.sasc.biopet.core.{ Reference, BiopetJavaCommandLineFunction }
import org.broadinstitute.gatk.engine.phonehome.GATKRunReport
import org.broadinstitute.gatk.queue.extensions.gatk.CommandLineGATK
trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction with Reference {
......@@ -20,6 +21,15 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction wit
if (config.contains("intervals")) intervals = config("intervals").asFileList
if (config.contains("exclude_intervals")) excludeIntervals = config("exclude_intervals").asFileList
Option(config("et").value) match {
case Some("NO_ET") => et = GATKRunReport.PhoneHomeOption.NO_ET
case Some("AWS") => et = GATKRunReport.PhoneHomeOption.AWS
case Some("STDOUT") => et = GATKRunReport.PhoneHomeOption.STDOUT
case Some(x) => throw new IllegalArgumentException(s"Unknown et option for gatk: $x")
case _ =>
}
if (config.contains("gatk_key")) gatk_key = config("gatk_key")
if (config.contains("pedigree")) pedigree = config("pedigree")
......
protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala
View file @ 1dd927a3
......@@ -93,6 +93,12 @@ class ShivaTest extends TestNGSuite with Matchers {
object ShivaTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): String = {
val file = new File(outputDir, "input" + File.separator + name)
Files.touch(file)
file.getAbsolutePath
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......@@ -136,8 +142,8 @@ object ShivaTest {
val sample1 = Map(
"samples" -> Map("sample1" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "1_1_R1.fq",
"R2" -> "1_1_R2.fq"
"R1" -> inputTouch("1_1_R1.fq"),
"R2" -> inputTouch("1_1_R2.fq")
)
)
)))
......@@ -145,8 +151,8 @@ object ShivaTest {
val sample2 = Map(
"samples" -> Map("sample2" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "2_1_R1.fq",
"R2" -> "2_1_R2.fq"
"R1" -> inputTouch("2_1_R1.fq"),
"R2" -> inputTouch("2_1_R2.fq")
)
)
)))
......@@ -154,12 +160,12 @@ object ShivaTest {
val sample3 = Map(
"samples" -> Map("sample3" -> Map("libraries" -> Map(
"lib1" -> Map(
"R1" -> "3_1_R1.fq",
"R2" -> "3_1_R2.fq"
"R1" -> inputTouch("3_1_R1.fq"),
"R2" -> inputTouch("3_1_R2.fq")
),
"lib2" -> Map(
"R1" -> "3_2_R1.fq",
"R2" -> "3_2_R2.fq"
"R1" -> inputTouch("3_2_R1.fq"),
"R2" -> inputTouch("3_2_R2.fq")
)
)
)))
......
protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala
View file @ 1dd927a3
......@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
val map = Map("variantcallers" -> callers.toList)
val pipeline = initPipeline(map)
pipeline.inputBams = (for (n <- 1 to bams) yield new File("bam_" + n + ".bam")).toList
pipeline.inputBams = (for (n <- 1 to bams) yield ShivaVariantcallingTest.inputTouch("bam_" + n + ".bam")).toList
val illegalArgumentException = pipeline.inputBams.isEmpty ||
(!raw && !bcftools &&
......@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
object ShivaVariantcallingTest {
val outputDir = Files.createTempDir()
new File(outputDir, "input").mkdirs()
def inputTouch(name: String): File = {
val file = new File(outputDir, "input" + File.separator + name).getAbsoluteFile
Files.touch(file)
file
}
private def copyFile(name: String): Unit = {
val is = getClass.getResourceAsStream("/" + name)
......