Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Mirrors
biopet.biopet
Commits
1dd927a3
Commit
1dd927a3
authored
Sep 22, 2015
by
Peter van 't Hof
Browse files
Merge remote-tracking branch 'origin/develop' into feature-pipeable
parents
f7ea04a7
f4063de9
Changes
62
Hide whitespace changes
Inline
Side-by-side
external-example/.gitignore
0 → 100644
View file @
1dd927a3
# Project-related
dependency-reduced-pom.xml
git.properties
# gedit
*~
# Vim
*.swp
# IntelliJ
.idea/*
*.iml
target/
public/target/
protected/target/
external-example/README.md
0 → 100644
View file @
1dd927a3
Test implementation of Magpie 2.0
\ No newline at end of file
external-example/pom.xml
0 → 100644
View file @
1dd927a3
<?xml version="1.0" encoding="UTF-8"?>
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<modelVersion>
4.0.0
</modelVersion>
<!--TODO: replace groupId -->
<groupId>
org.example.group
</groupId>
<!--TODO: replace artifactId -->
<artifactId>
ExternalExample
</artifactId>
<!--TODO: replace version, for a new pipeline it's advised to start with '0.1.0-SNAPSHOT' -->
<version>
0.1.0-SNAPSHOT
</version>
<properties>
<project.build.sourceEncoding>
UTF-8
</project.build.sourceEncoding>
<scoverage.plugin.version>
1.0.4
</scoverage.plugin.version>
<sting.shade.phase>
package
</sting.shade.phase>
<!--
TODO: replace app.main.class, this is the class that get executed when running the jar file
This can be any executable that have a main method. In Biopet any pipeline can be used as direct executable.
Value for direct pipeline: 'org.example.group.pipelines.SimplePipeline'
In the given example is an extension of the biopet executable. In this example there are multiple pipelines in 1 executable.
It's also possible to make your own main function and call the main function with it's argument of the pipeline from there.
-->
<app.main.class>
org.example.group.ExecutableExample
</app.main.class>
</properties>
<dependencies>
<!--
In here maven dependencies can be placed, when importing a biopet pipeline 'Biopet-Framework' is not required.
When only using the framework without pipeline you need to import BiopetFramework.
It's advised to not use different versions of the pipeline and the framework.
-->
<dependency>
<groupId>
nl.lumc.sasc
</groupId>
<artifactId>
BiopetCore
</artifactId>
<!--TODO: replace version of pipeline to a fixed version -->
<version>
0.5.0-SNAPSHOT
</version>
</dependency>
<dependency>
<groupId>
nl.lumc.sasc
</groupId>
<artifactId>
BiopetExtensions
</artifactId>
<version>
0.5.0-SNAPSHOT
</version>
</dependency>
<dependency>
<groupId>
nl.lumc.sasc
</groupId>
<artifactId>
Shiva
</artifactId>
<!--TODO: replace version of pipeline to a fixed version -->
<version>
0.5.0-SNAPSHOT
</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>
${basedir}/src/main/scala
</sourceDirectory>
<testSourceDirectory>
${basedir}/src/test/scala
</testSourceDirectory>
<testResources>
<testResource>
<directory>
${basedir}/src/test/resources
</directory>
<includes>
<include>
**/*
</include>
</includes>
</testResource>
</testResources>
<resources>
<resource>
<directory>
${basedir}/src/main/resources
</directory>
<includes>
<include>
**/*
</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-shade-plugin
</artifactId>
<version>
2.4.1
</version>
<configuration>
<!--suppress MavenModelInspection -->
<finalName>
${project.artifactId}-${project.version}-${git.commit.id.abbrev}
</finalName>
<transformers>
<transformer
implementation=
"org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"
>
<manifestEntries>
<Main-Class>
${app.main.class}
</Main-Class>
<!--suppress MavenModelInspection -->
<X-Compile-Source-JDK>
${maven.compile.source}
</X-Compile-Source-JDK>
<!--suppress MavenModelInspection -->
<X-Compile-Target-JDK>
${maven.compile.target}
</X-Compile-Target-JDK>
</manifestEntries>
</transformer>
</transformers>
<filters>
</filters>
</configuration>
<executions>
<execution>
<phase>
package
</phase>
<goals>
<goal>
shade
</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-surefire-plugin
</artifactId>
<version>
2.18.1
</version>
<configuration>
<forkCount>
1C
</forkCount>
<workingDirectory>
${project.build.directory}
</workingDirectory>
</configuration>
</plugin>
<plugin>
<artifactId>
maven-dependency-plugin
</artifactId>
<version>
2.10
</version>
<executions>
<execution>
<id>
copy-installed
</id>
<phase>
prepare-package
</phase>
<goals>
<goal>
list
</goal>
</goals>
<configuration>
<outputFile>
${project.build.outputDirectory}/dependency_list.txt
</outputFile>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>
net.alchim31.maven
</groupId>
<artifactId>
scala-maven-plugin
</artifactId>
<version>
3.2.0
</version>
<executions>
<execution>
<id>
scala-compile
</id>
<goals>
<goal>
compile
</goal>
<goal>
testCompile
</goal>
</goals>
<configuration>
<args>
<arg>
-dependencyfile
</arg>
<arg>
${project.build.directory}/.scala_dependencies
</arg>
<arg>
-deprecation
</arg>
<arg>
-feature
</arg>
</args>
</configuration>
</execution>
</executions>
<!-- ... (see other usage or goals for details) ... -->
</plugin>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-jar-plugin
</artifactId>
<version>
2.5
</version>
<executions>
<execution>
<goals>
<goal>
test-jar
</goal>
</goals>
</execution>
</executions>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>
true
</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>
true
</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-compiler-plugin
</artifactId>
<version>
2.3.2
</version>
<configuration>
<showDeprecation>
true
</showDeprecation>
</configuration>
</plugin>
<plugin>
<groupId>
org.scalariform
</groupId>
<artifactId>
scalariform-maven-plugin
</artifactId>
<version>
0.1.4
</version>
<executions>
<execution>
<phase>
process-sources
</phase>
<goals>
<goal>
format
</goal>
</goals>
<configuration>
<rewriteArrowSymbols>
false
</rewriteArrowSymbols>
<alignParameters>
true
</alignParameters>
<alignSingleLineCaseStatements_maxArrowIndent>
40
</alignSingleLineCaseStatements_maxArrowIndent>
<alignSingleLineCaseStatements>
true
</alignSingleLineCaseStatements>
<compactStringConcatenation>
false
</compactStringConcatenation>
<compactControlReadability>
false
</compactControlReadability>
<doubleIndentClassDeclaration>
false
</doubleIndentClassDeclaration>
<formatXml>
true
</formatXml>
<indentLocalDefs>
false
</indentLocalDefs>
<indentPackageBlocks>
true
</indentPackageBlocks>
<indentSpaces>
2
</indentSpaces>
<placeScaladocAsterisksBeneathSecondAsterisk>
false
</placeScaladocAsterisksBeneathSecondAsterisk>
<preserveDanglingCloseParenthesis>
true
</preserveDanglingCloseParenthesis>
<preserveSpaceBeforeArguments>
false
</preserveSpaceBeforeArguments>
<rewriteArrowSymbols>
false
</rewriteArrowSymbols>
<spaceBeforeColon>
false
</spaceBeforeColon>
<spaceInsideBrackets>
false
</spaceInsideBrackets>
<spaceInsideParentheses>
false
</spaceInsideParentheses>
<spacesWithinPatternBinders>
true
</spacesWithinPatternBinders>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>
pl.project13.maven
</groupId>
<artifactId>
git-commit-id-plugin
</artifactId>
<version>
2.1.10
</version>
<executions>
<execution>
<goals>
<goal>
revision
</goal>
</goals>
</execution>
</executions>
<configuration>
<prefix>
git
</prefix>
<dateFormat>
dd.MM.yyyy '@' HH:mm:ss z
</dateFormat>
<verbose>
false
</verbose>
<!-- TODO: This directory need to be changed depening where your .git folder is relative from this pom.xml -->
<dotGitDirectory>
${basedir}/../.git
</dotGitDirectory>
<useNativeGit>
true
</useNativeGit>
<skipPoms>
false
</skipPoms>
<generateGitPropertiesFile>
true
</generateGitPropertiesFile>
<generateGitPropertiesFilename>
src/main/resources/git.properties
</generateGitPropertiesFilename>
<failOnNoGitDirectory>
false
</failOnNoGitDirectory>
<abbrevLength>
8
</abbrevLength>
<skip>
false
</skip>
<gitDescribe>
<skip>
false
</skip>
<always>
false
</always>
<abbrev>
8
</abbrev>
<dirty>
-dirty
</dirty>
<forceLongFormat>
false
</forceLongFormat>
</gitDescribe>
</configuration>
</plugin>
<plugin>
<groupId>
com.mycila
</groupId>
<artifactId>
license-maven-plugin
</artifactId>
<version>
2.6
</version>
<configuration>
<excludes>
<exclude>
**/*git*
</exclude>
<exclude>
**/License*
</exclude>
<exclude>
**/*.bam
</exclude>
<exclude>
**/*.bai
</exclude>
<exclude>
**/*.gtf
</exclude>
<exclude>
**/*.fq
</exclude>
<exclude>
**/*.sam
</exclude>
<exclude>
**/*.bed
</exclude>
<exclude>
**/*.refFlat
</exclude>
<exclude>
**/*.R
</exclude>
<exclude>
**/*.rscript
</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>
org.scoverage
</groupId>
<artifactId>
scoverage-maven-plugin
</artifactId>
<version>
${scoverage.plugin.version}
</version>
<configuration>
<scalaVersion>
2.10.2
</scalaVersion>
<!-- other parameters -->
</configuration>
</plugin>
</plugins>
</build>
<reporting>
<plugins>
<plugin>
<groupId>
org.scoverage
</groupId>
<artifactId>
scoverage-maven-plugin
</artifactId>
<version>
${scoverage.plugin.version}
</version>
</plugin>
</plugins>
</reporting>
</project>
\ No newline at end of file
external-example/src/main/scala/org/example/group/ExecutableExample.scala
0 → 100644
View file @
1dd927a3
package
org.example.group
import
nl.lumc.sasc.biopet.utils.
{
BiopetExecutable
,
MainCommand
}
/**
* Created by pjvanthof on 30/08/15.
*/
object
ExecutableExample
extends
BiopetExecutable
{
/** This list defines the pipeline that are usable from the executable */
def
pipelines
:
List
[
MainCommand
]
=
List
(
org
.
example
.
group
.
pipelines
.
MultisamplePipeline
,
org
.
example
.
group
.
pipelines
.
BiopetPipeline
,
org
.
example
.
group
.
pipelines
.
SimplePipeline
)
/** This list defines the (biopet)tools that are usable from the executable */
def
tools
:
List
[
MainCommand
]
=
Nil
}
external-example/src/main/scala/org/example/group/pipelines/BiopetPipeline.scala
0 → 100644
View file @
1dd927a3
package
org.example.group.pipelines
import
nl.lumc.sasc.biopet.core.PipelineCommand
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
nl.lumc.sasc.biopet.core.summary.SummaryQScript
import
nl.lumc.sasc.biopet.pipelines.shiva.Shiva
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvan_thof on 8/28/15.
*/
//TODO: Replace class Name
class
BiopetPipeline
(
val
root
:
Configurable
)
extends
QScript
with
SummaryQScript
{
def
this
()
=
this
(
null
)
/** Only required when using [[SummaryQScript]] */
def
summaryFile
=
new
File
(
outputDir
,
"magpie.summary.json"
)
/** Only required when using [[SummaryQScript]] */
def
summaryFiles
:
Map
[
String
,
File
]
=
Map
()
/** Only required when using [[SummaryQScript]] */
def
summarySettings
=
Map
()
// This method can be used to initialize some classes where needed
def
init
()
:
Unit
=
{
}
// This method is the actual pipeline
def
biopetScript
:
Unit
=
{
// Executing a biopet pipeline inside
val
shiva
=
new
Shiva
(
this
)
shiva
.
init
()
shiva
.
biopetScript
()
addAll
(
shiva
.
functions
)
/* Only required when using [[SummaryQScript]] */
addSummaryQScript
(
shiva
)
// From here you can use the output files of shiva as input file of other jobs
}
}
//TODO: Replace object Name, must be the same as the class of the pipeline
object
BiopetPipeline
extends
PipelineCommand
external-example/src/main/scala/org/example/group/pipelines/MultisamplePipeline.scala
0 → 100644
View file @
1dd927a3
package
org.example.group.pipelines
import
nl.lumc.sasc.biopet.core.
{
PipelineCommand
,
MultiSampleQScript
}
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvanthof on 30/08/15.
*/
class
MultisamplePipeline
(
val
root
:
Configurable
)
extends
QScript
with
MultiSampleQScript
{
qscript
=>
def
this
()
=
this
(
null
)
def
init
:
Unit
=
{
}
def
biopetScript
:
Unit
=
{
addSamplesJobs
()
// This executes jobs for all samples
}
def
addMultiSampleJobs
:
Unit
=
{
// this code will be executed after all code of all samples is executed
}
def
summaryFile
:
File
=
new
File
(
outputDir
,
"MultisamplePipeline.summary.json"
)
//TODO: Add summary
def
summaryFiles
:
Map
[
String
,
File
]
=
Map
()
//TODO: Add summary
def
summarySettings
:
Map
[
String
,
Any
]
=
Map
()
def
makeSample
(
id
:
String
)
=
new
Sample
(
id
)
class
Sample
(
sampleId
:
String
)
extends
AbstractSample
(
sampleId
)
{
def
makeLibrary
(
id
:
String
)
=
new
Library
(
id
)
class
Library
(
libId
:
String
)
extends
AbstractLibrary
(
libId
)
{
//TODO: Add summary
def
summaryFiles
:
Map
[
String
,
File
]
=
Map
()
//TODO: Add summary
def
summaryStats
:
Map
[
String
,
Any
]
=
Map
()
def
addJobs
:
Unit
=
{
//TODO: add library specific jobs
}
}
//TODO: Add summary
def
summaryFiles
:
Map
[
String
,
File
]
=
Map
()
//TODO: Add summary
def
summaryStats
:
Map
[
String
,
Any
]
=
Map
()
def
addJobs
:
Unit
=
{
addPerLibJobs
()
// This add jobs for each library
//TODO: add sample specific jobs
}
}
}
object
MultisamplePipeline
extends
PipelineCommand
\ No newline at end of file
external-example/src/main/scala/org/example/group/pipelines/SimplePipeline.scala
0 → 100644
View file @
1dd927a3
package
org.example.group.pipelines
import
nl.lumc.sasc.biopet.core.
{
BiopetQScript
,
PipelineCommand
}
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
nl.lumc.sasc.biopet.extensions.
{
Gzip
,
Cat
}
import
org.broadinstitute.gatk.queue.QScript
/**
* Created by pjvanthof on 30/08/15.
*/
//TODO: Replace class name, must be the same as the class of the pipeline
class
SimplePipeline
(
val
root
:
Configurable
)
extends
QScript
with
BiopetQScript
{
// A constructor without arguments is needed if this pipeline is a root pipeline
def
this
()
=
this
(
null
)
@Input
(
required
=
true
)
var
inputFile
:
File
=
null
/** This method can be used to initialize some classes where needed */
def
init
()
:
Unit
=
{
}
/** This method is the actual pipeline */
def
biopetScript
:
Unit
=
{
val
cat
=
new
Cat
(
this
)
cat
.
input
:+=
inputFile
cat
.
output
=
new
File
(
outputDir
,
"file.out"
)
add
(
cat
)
val
gzip
=
new
Gzip
(
this
)
gzip
.
input
:+=
cat
.
output
gzip
.
output
=
new
File
(
outputDir
,
"file.out.gz"
)
add
(
gzip
)
}
}
//TODO: Replace object name, must be the same as the class of the pipeline
object
SimplePipeline
extends
PipelineCommand
pom.xml
View file @
1dd927a3
...
...
@@ -16,5 +16,6 @@
<modules>
<module>
public
</module>
<module>
protected
</module>
<module>
external-example
</module>
</modules>
</project>
protected/biopet-gatk-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/broad/GatkGeneral.scala
View file @
1dd927a3
...
...
@@ -5,7 +5,8 @@
*/
package
nl.lumc.sasc.biopet.extensions.gatk.broad
import
nl.lumc.sasc.biopet.core.
{
BiopetJavaCommandLineFunction
,
Reference
}
import
nl.lumc.sasc.biopet.core.
{
Reference
,
BiopetJavaCommandLineFunction
}
import
org.broadinstitute.gatk.engine.phonehome.GATKRunReport
import
org.broadinstitute.gatk.queue.extensions.gatk.CommandLineGATK
trait
GatkGeneral
extends
CommandLineGATK
with
BiopetJavaCommandLineFunction
with
Reference
{
...
...
@@ -20,6 +21,15 @@ trait GatkGeneral extends CommandLineGATK with BiopetJavaCommandLineFunction wit
if
(
config
.
contains
(
"intervals"
))
intervals
=
config
(
"intervals"
).
asFileList
if
(
config
.
contains
(
"exclude_intervals"
))
excludeIntervals
=
config
(
"exclude_intervals"
).
asFileList
Option
(
config
(
"et"
).
value
)
match
{
case
Some
(
"NO_ET"
)
=>
et
=
GATKRunReport
.
PhoneHomeOption
.
NO_ET
case
Some
(
"AWS"
)
=>
et
=
GATKRunReport
.
PhoneHomeOption
.
AWS
case
Some
(
"STDOUT"
)
=>
et
=
GATKRunReport
.
PhoneHomeOption
.
STDOUT
case
Some
(
x
)
=>
throw
new
IllegalArgumentException
(
s
"Unknown et option for gatk: $x"
)
case
_
=>
}
if
(
config
.
contains
(
"gatk_key"
))
gatk_key
=
config
(
"gatk_key"
)
if
(
config
.
contains
(
"pedigree"
))
pedigree
=
config
(
"pedigree"
)
...
...
protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaTest.scala
View file @
1dd927a3
...
...
@@ -93,6 +93,12 @@ class ShivaTest extends TestNGSuite with Matchers {
object
ShivaTest
{
val
outputDir
=
Files
.
createTempDir
()
new
File
(
outputDir
,
"input"
).
mkdirs
()
def
inputTouch
(
name
:
String
)
:
String
=
{
val
file
=
new
File
(
outputDir
,
"input"
+
File
.
separator
+
name
)
Files
.
touch
(
file
)
file
.
getAbsolutePath
}
private
def
copyFile
(
name
:
String
)
:
Unit
=
{
val
is
=
getClass
.
getResourceAsStream
(
"/"
+
name
)
...
...
@@ -136,8 +142,8 @@ object ShivaTest {
val
sample1
=
Map
(
"samples"
->
Map
(
"sample1"
->
Map
(
"libraries"
->
Map
(
"lib1"
->
Map
(
"R1"
->
"1_1_R1.fq"
,
"R2"
->
"1_1_R2.fq"
"R1"
->
inputTouch
(
"1_1_R1.fq"
)
,
"R2"
->
inputTouch
(
"1_1_R2.fq"
)
)
)
)))
...
...
@@ -145,8 +151,8 @@ object ShivaTest {
val
sample2
=
Map
(
"samples"
->
Map
(
"sample2"
->
Map
(
"libraries"
->
Map
(
"lib1"
->
Map
(
"R1"
->
"2_1_R1.fq"
,
"R2"
->
"2_1_R2.fq"
"R1"
->
inputTouch
(
"2_1_R1.fq"
)
,
"R2"
->
inputTouch
(
"2_1_R2.fq"
)
)
)
)))
...
...
@@ -154,12 +160,12 @@ object ShivaTest {
val
sample3
=
Map
(
"samples"
->
Map
(
"sample3"
->
Map
(
"libraries"
->
Map
(
"lib1"
->
Map
(
"R1"
->
"3_1_R1.fq"
,
"R2"
->
"3_1_R2.fq"
"R1"
->
inputTouch
(
"3_1_R1.fq"
)
,
"R2"
->
inputTouch
(
"3_1_R2.fq"
)
),
"lib2"
->
Map
(
"R1"
->
"3_2_R1.fq"
,
"R2"
->
"3_2_R2.fq"
"R1"
->
inputTouch
(
"3_2_R1.fq"
)
,
"R2"
->
inputTouch
(
"3_2_R2.fq"
)
)
)
)))
...
...
protected/biopet-gatk-pipelines/src/test/scala/nl/lumc/sasc/biopet/pipelines/gatk/ShivaVariantcallingTest.scala
View file @
1dd927a3
...
...
@@ -73,7 +73,7 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
val
map
=
Map
(
"variantcallers"
->
callers
.
toList
)
val
pipeline
=
initPipeline
(
map
)
pipeline
.
inputBams
=
(
for
(
n
<-
1
to
bams
)
yield
new
File
(
"bam_"
+
n
+
".bam"
)).
toList
pipeline
.
inputBams
=
(
for
(
n
<-
1
to
bams
)
yield
ShivaVariantcallingTest
.
inputTouch
(
"bam_"
+
n
+
".bam"
)).
toList
val
illegalArgumentException
=
pipeline
.
inputBams
.
isEmpty
||
(!
raw
&&
!
bcftools
&&
...
...
@@ -107,6 +107,12 @@ class ShivaVariantcallingTest extends TestNGSuite with Matchers {
object
ShivaVariantcallingTest
{
val
outputDir
=
Files
.
createTempDir
()
new
File
(
outputDir
,
"input"
).
mkdirs
()
def
inputTouch
(
name
:
String
)
:
File
=
{
val
file
=
new
File
(
outputDir
,
"input"
+
File
.
separator
+
name
).
getAbsoluteFile
Files
.
touch
(
file
)
file
}
private
def
copyFile
(
name
:
String
)
:
Unit
=
{
val
is
=
getClass
.
getResourceAsStream
(
"/"
+
name
)
...
...
protected/biopet-protected-package/src/main/scala/nl/lumc/sasc/biopet/BiopetExecutableProtected.scala
View file @
1dd927a3
...
...
@@ -8,7 +8,7 @@ package nl.lumc.sasc.biopet
import
nl.lumc.sasc.biopet.utils.
{
BiopetExecutable
,
MainCommand
}
object
BiopetExecutableProtected
extends
BiopetExecutable
{
def
pipelines
:
List
[
MainCommand
]
=
BiopetExecutablePublic
.
pipelines
:::
List
(