Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Mirrors
biopet.biopet
Commits
6f3ac600
Commit
6f3ac600
authored
Apr 21, 2017
by
Peter van 't Hof
Committed by
GitHub
Apr 21, 2017
Browse files
Merge pull request #88 from biopet/fix-BIOPET-645
Corrections on threads
parents
42e2ffe3
684a0963
Changes
13
Hide whitespace changes
Inline
Side-by-side
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala
View file @
6f3ac600
...
...
@@ -67,6 +67,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
val
skipWriteDependencies
:
Boolean
=
config
(
"skip_write_dependencies"
,
default
=
false
)
val
writeHtmlReport
:
Boolean
=
config
(
"write_html_report"
,
default
=
true
)
/** Script from queue itself, final to force some checks for each pipeline and write report */
final
def
script
()
{
outputDir
=
config
(
"output_dir"
)
...
...
@@ -126,8 +128,6 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
}
})
val
writeHtmlReport
:
Boolean
=
config
(
"write_html_report"
,
default
=
true
)
if
(
writeHtmlReport
)
{
logger
.
info
(
"Adding report"
)
this
match
{
...
...
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/CommandLineResources.scala
View file @
6f3ac600
...
...
@@ -22,7 +22,16 @@ import org.broadinstitute.gatk.queue.function.CommandLineFunction
*/
trait
CommandLineResources
extends
CommandLineFunction
with
Configurable
{
/**
* This value is overruling threads method when this is set.
* This can be used to limit the number of threads on a global level.
*/
lazy
val
maxThreads
:
Option
[
Int
]
=
config
(
"max_threads"
)
/** To set an other default threads this method need to be override */
def
defaultThreads
=
1
/** This method will get and set the cores requested */
final
def
threads
=
nCoresRequest
match
{
case
Some
(
i
)
=>
i
case
_
=>
...
...
@@ -72,7 +81,6 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
* @return number of threads
*/
private
def
getThreads
(
default
:
Int
)
:
Int
=
{
val
maxThreads
:
Option
[
Int
]
=
config
(
"maxthreads"
)
val
threads
:
Int
=
config
(
"threads"
,
default
=
default
)
maxThreads
match
{
case
Some
(
max
)
=>
if
(
max
>
threads
)
threads
else
max
...
...
@@ -120,11 +128,12 @@ trait CommandLineResources extends CommandLineFunction with Configurable {
protected
def
combineResources
(
commands
:
List
[
CommandLineResources
])
:
Unit
=
{
commands
.
foreach
(
_
.
setResources
())
nCoresRequest
=
Some
(
commands
.
map
(
_
.
threads
).
sum
+
threadsCorrection
)
nCoresRequest
=
nCoresRequest
.
map
(
x
=>
if
(
x
>
maxThreads
.
getOrElse
(
x
))
maxThreads
.
getOrElse
(
x
)
else
x
)
_coreMemory
=
commands
.
map
(
cmd
=>
cmd
.
coreMemory
*
(
cmd
.
threads
.
toDouble
/
threads
.
toDouble
)).
sum
memoryLimit
=
Some
(
_coreMemory
*
threads
)
residentLimit
=
Some
((
_coreMemory
+
(
0.5
*
retry
))
*
residentFactor
*
(
if
(
multiplyRssThreads
)
threads
else
1
))
vmem
=
Some
((
_coreMemory
*
(
vmemFactor
+
(
0.5
*
retry
))
*
(
if
(
multiplyVmemThreads
)
threads
else
1
))
+
"G"
)
memoryLimit
=
Some
(
_coreMemory
*
nCoresRequest
.
getOrElse
(
threads
)
)
residentLimit
=
Some
((
_coreMemory
+
(
0.5
*
retry
))
*
residentFactor
*
(
if
(
multiplyRssThreads
)
nCoresRequest
.
getOrElse
(
threads
)
else
1
))
vmem
=
Some
((
_coreMemory
*
(
vmemFactor
+
(
0.5
*
retry
))
*
(
if
(
multiplyVmemThreads
)
nCoresRequest
.
getOrElse
(
threads
)
else
1
))
+
"G"
)
}
}
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/summary/SummaryQScript.scala
View file @
6f3ac600
...
...
@@ -135,6 +135,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
}
md5sum
.
input
=
file
md5sum
.
output
=
new
File
(
file
.
getParentFile
,
file
.
getName
+
".md5"
)
md5sum
.
jobOutputFile
=
new
File
(
file
.
getParentFile
,
s
".${file.getName}.md5.md5sum.out"
)
// Need to not write a md5 file outside the outputDir
if
(!
file
.
getAbsolutePath
.
startsWith
(
outputDir
.
getAbsolutePath
))
...
...
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/CommandLineResourcesTest.scala
View file @
6f3ac600
...
...
@@ -64,7 +64,7 @@ class CommandLineResourcesTest extends TestNGSuite with Matchers {
@Test
def
testMaxThreads
()
:
Unit
=
{
val
cmd
=
new
CommandLineFunctionMock
(
Map
(
"maxthreads"
->
5
,
"threads"
->
10
))
with
CommandLineResources
val
cmd
=
new
CommandLineFunctionMock
(
Map
(
"max
_
threads"
->
5
,
"threads"
->
10
))
with
CommandLineResources
cmd
.
threads
shouldBe
5
}
...
...
biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/freec/FreeC.scala
View file @
6f3ac600
...
...
@@ -84,7 +84,6 @@ class FreeC(val parent: Configurable) extends BiopetCommandLineFunction with Ref
var
minExpectedGC
:
Option
[
Double
]
=
config
(
"minExpectedGC"
)
var
maxExpectedGC
:
Option
[
Double
]
=
config
(
"maxExpectedGC"
)
var
minimalSubclonePresence
:
Option
[
Int
]
=
config
(
"minimalSubclonePresence"
)
var
maxThreads
:
Int
=
getThreads
var
noisyData
:
Boolean
=
config
(
"noisyData"
,
default
=
false
)
//var outputDir: File
...
...
biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/VcfStatsForSv.scala
View file @
6f3ac600
...
...
@@ -36,7 +36,8 @@ class VcfStatsForSv(val parent: Configurable) extends ToolCommandFunction with S
@Output
(
required
=
true
)
var
outputFile
:
File
=
_
override
def
defaultCoreMemory
=
1.0
override
def
defaultResidentFactor
=
2.0
override
def
defaultVmemFactor
=
3.0
override
def
cmdLine
=
super
.
cmdLine
+
required
(
"-i"
,
inputFile
)
+
...
...
biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/vcfstats/VcfStatsForSv.scala
View file @
6f3ac600
...
...
@@ -38,8 +38,7 @@ object VcfStatsForSv extends ToolCommand {
val
stats
:
Map
[
String
,
Any
]
=
getVariantCounts
(
cmdArgs
.
inputFile
,
cmdArgs
.
histBinBoundaries
)
ConfigUtils
.
mapToYamlFile
(
stats
,
cmdArgs
.
outputFile
)
ConfigUtils
.
mapToJsonFile
(
stats
,
cmdArgs
.
outputFile
)
}
/** Parses a vcf-file and counts sv-s by type and size. Sv-s are divided to different size classes, the parameter histogramBinBoundaries gives the boundaries between these classes. */
...
...
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/ConfigUtils.scala
View file @
6f3ac600
...
...
@@ -168,6 +168,12 @@ object ConfigUtils extends Logging {
writer
.
close
()
}
def
mapToJsonFile
(
map
:
Map
[
String
,
Any
],
outputFile
:
File
)
=
{
val
writer
=
new
PrintWriter
(
outputFile
)
writer
.
println
(
anyToJson
(
map
).
toString
())
writer
.
close
()
}
/** Convert json to native scala map/values */
def
jsonToMap
(
json
:
Json
)
:
Map
[
String
,
Any
]
=
{
var
output
:
Map
[
String
,
Any
]
=
Map
()
...
...
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/intervals/BedRecordList.scala
View file @
6f3ac600
...
...
@@ -113,7 +113,7 @@ case class BedRecordList(val chrRecords: Map[String, List[BedRecord]], val heade
}
/** This return the fraction of the regions comparing to a length */
def
fractionOf
(
length
:
Long
)
:
Double
=
length
.
toDouble
/
length
def
fractionOf
(
length
:
Long
)
:
Double
=
this
.
length
.
toDouble
/
length
.
toDouble
/** This return the fraction of the regions comparing to a reference */
def
fractionOfReference
(
dict
:
SAMSequenceDictionary
)
:
Double
=
fractionOf
(
dict
.
getReferenceLength
)
...
...
flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala
View file @
6f3ac600
...
...
@@ -41,6 +41,8 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
/** Make a final fastq files, by default only when flexiprep is the main pipeline */
var
keepQcFastqFiles
:
Boolean
=
config
(
"keepQcFastqFiles"
,
default
=
parent
==
null
)
override
def
defaults
=
super
.
defaults
++
Map
(
"max_threads"
->
4
)
/** Returns files to store in summary */
def
summaryFiles
:
Map
[
String
,
File
]
=
{
Map
(
"input_R1"
->
inputR1
,
"output_R1"
->
fastqR1Qc
)
++
...
...
@@ -226,6 +228,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
override
def
summaryDeps
:
List
[
File
]
=
qcCmdR1
.
summaryDeps
:::
qcCmdR2
.
summaryDeps
:::
super
.
summaryDeps
}
pipe
.
jobOutputFile
=
new
File
(
outDir
,
".qc_cmd.out"
)
pipe
.
deps
::=
fastqcR1
.
output
pipe
.
deps
::=
fastqcR2
.
output
pipe
.
deps
::=
R1_in
...
...
@@ -241,6 +244,7 @@ class Flexiprep(val parent: Configurable) extends QScript with SummaryQScript wi
R2
=
Some
(
fqSync
.
outputFastq2
)
}
else
{
qcCmdR1
.
nCoresRequest
=
Some
(
2
)
qcCmdR1
.
jobOutputFile
=
new
File
(
outDir
,
".qc_cmd.out"
)
add
(
qcCmdR1
)
R1
=
qcCmdR1
.
output
}
...
...
mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/Mapping.scala
View file @
6f3ac600
...
...
@@ -86,6 +86,9 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
/** Readgroup sequencing center */
protected
var
readgroupSequencingCenter
:
Option
[
String
]
=
config
(
"readgroup_sequencing_center"
)
/** Readgroup library */
protected
var
readgroupLibrary
:
Option
[
String
]
=
config
(
"readgroup_library"
)
/** Readgroup description */
protected
var
readgroupDescription
:
Option
[
String
]
=
config
(
"readgroup_description"
)
...
...
@@ -572,7 +575,7 @@ class Mapping(val parent: Configurable) extends QScript with SummaryQScript with
/** Returns readgroup for bwa */
def
getReadGroupBwa
:
String
=
{
var
RG
:
String
=
"@RG\\t"
+
"ID:"
+
readgroupId
+
"\\t"
RG
+=
"LB:"
+
l
ibId
.
get
+
"\\t"
readgroupLibrary
.
foreach
(
lb
=>
RG
+=
"LB:"
+
l
b
+
"\\t"
)
RG
+=
"PL:"
+
platform
+
"\\t"
platformUnit
.
foreach
(
x
=>
RG
+=
"PU:"
+
x
+
"\\t"
)
RG
+=
"SM:"
+
sampleId
.
get
+
"\\t"
...
...
shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/Shiva.scala
View file @
6f3ac600
...
...
@@ -44,12 +44,6 @@ class Shiva(val parent: Configurable) extends QScript with MultisampleMappingTra
Some
(
shiva
)
}
override
def
defaults
=
Map
(
"haplotypecaller"
->
Map
(
"stand_call_conf"
->
30
,
"stand_emit_conf"
->
0
),
"genotypegvcfs"
->
Map
(
"stand_call_conf"
->
30
,
"stand_emit_conf"
->
0
),
"unifiedgenotyper"
->
Map
(
"stand_call_conf"
->
30
,
"stand_emit_conf"
->
0
)
)
lazy
val
usePrintReads
:
Boolean
=
config
(
"use_printreads"
,
default
=
true
)
/** Method to make the variantcalling namespace of shiva */
...
...
shiva/src/main/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaSvCalling.scala
View file @
6f3ac600
...
...
@@ -20,9 +20,14 @@ import nl.lumc.sasc.biopet.extensions.Pysvtools
import
nl.lumc.sasc.biopet.extensions.tools.VcfStatsForSv
import
nl.lumc.sasc.biopet.pipelines.shiva.svcallers._
import
nl.lumc.sasc.biopet.utils.config.Configurable
import
nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import
nl.lumc.sasc.biopet.utils.
{
BamUtils
,
Logging
}
import
org.broadinstitute.gatk.queue.QScript
import
scala.concurrent.Await
import
scala.concurrent.duration.Duration
import
scala.concurrent.ExecutionContext.Implicits.global
/**
* Common trait for ShivaVariantcalling
*
...
...
@@ -43,7 +48,16 @@ class ShivaSvCalling(val parent: Configurable) extends QScript with SummaryQScri
/** Executed before script */
def
init
()
:
Unit
=
{
if
(
inputBamsArg
.
nonEmpty
)
inputBams
=
BamUtils
.
sampleBamMap
(
inputBamsArg
)
if
(
inputBamsArg
.
nonEmpty
)
{
inputBams
=
BamUtils
.
sampleBamMap
(
inputBamsArg
)
val
db
=
SummaryDb
.
openSqliteSummary
(
summaryDbFile
)
for
(
sampleName
<-
inputBams
.
keys
)
{
if
(
Await
.
result
(
db
.
getSampleId
(
summaryRunId
,
sampleName
),
Duration
.
Inf
).
isEmpty
)
{
db
.
createSample
(
sampleName
,
summaryRunId
)
}
}
}
outputMergedVCF
=
new
File
(
outputDir
,
"allsamples.merged.vcf"
)
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment