Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Mirrors
biopet.biopet
Commits
40b3b42d
Commit
40b3b42d
authored
Nov 26, 2016
by
Peter van 't Hof
Browse files
Added a jobs file
parent
130a7376
Changes
3
Hide whitespace changes
Inline
Side-by-side
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala
View file @
40b3b42d
...
...
@@ -121,10 +121,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
}
})
WriteDependencies
.
writeDependencies
(
functions
,
new
File
(
outputDir
,
s
".log/${qSettings.runName}.deps.json"
),
Some
(
new
File
(
outputDir
,
s
".log/${qSettings.runName}.mainJobs.json"
))
)
WriteDependencies
.
writeDependencies
(
functions
,
new
File
(
outputDir
,
".log"
),
qSettings
.
runName
)
Logging
.
checkErrors
()
logger
.
info
(
"Script complete without errors"
)
...
...
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/WriteDependencies.scala
View file @
40b3b42d
...
...
@@ -47,9 +47,10 @@ object WriteDependencies extends Logging with Configurable {
* This method will generate a json file where information about job and file dependencies are stored
*
* @param functions This should be all functions that are given to the graph of Queue
* @param outputFile Json file to write dependencies to
* @param outputDir
* @param prefix prefix
*/
def
writeDependencies
(
functions
:
Seq
[
QFunction
],
output
File
:
File
,
mainJobsFile
:
Option
[
File
]
=
None
)
:
Unit
=
{
def
writeDependencies
(
functions
:
Seq
[
QFunction
],
output
Dir
:
File
,
prefix
:
String
)
:
Unit
=
{
logger
.
info
(
"Start calculating dependencies"
)
val
errorOnMissingInput
:
Boolean
=
config
(
"error_on_missing_input"
,
false
)
...
...
@@ -126,17 +127,7 @@ object WriteDependencies extends Logging with Configurable {
"fail_at_start"
->
f
.
isFail
)
}.
toIterator
.
toMap
mainJobsFile
.
foreach
{
file
=>
val
mainJobs
=
jobs
.
filter
(
_
.
_2
(
"main_job"
)
==
true
).
map
{
case
(
name
,
job
)
=>
name
->
getMainDependencies
(
name
,
jobs
)
}
val
writer
=
new
PrintWriter
(
file
)
writer
.
println
(
ConfigUtils
.
mapToJson
(
mainJobs
).
spaces2
)
writer
.
close
()
}
val
outputFile
=
new
File
(
outputDir
,
s
"$prefix.deps.json"
)
logger
.
info
(
s
"Writing dependencies to: $outputFile"
)
val
writer
=
new
PrintWriter
(
outputFile
)
writer
.
println
(
ConfigUtils
.
mapToJson
(
Map
(
...
...
@@ -145,6 +136,20 @@ object WriteDependencies extends Logging with Configurable {
)).
spaces2
)
writer
.
close
()
val
jobsDeps
=
jobs
.
map
(
x
=>
x
.
_1
->
(
x
.
_2
(
"depends_on_jobs"
)))
val
jobsWriter
=
new
PrintWriter
(
new
File
(
outputDir
,
s
"$prefix.jobs.json"
))
jobsWriter
.
println
(
ConfigUtils
.
mapToJson
(
jobsDeps
).
spaces2
)
jobsWriter
.
close
()
val
mainJobs
=
jobs
.
filter
(
_
.
_2
(
"main_job"
)
==
true
).
map
{
case
(
name
,
job
)
=>
name
->
getMainDependencies
(
name
,
jobs
)
}
val
mainJobsWriter
=
new
PrintWriter
(
new
File
(
outputDir
,
s
"$prefix.main_jobs.json"
))
mainJobsWriter
.
println
(
ConfigUtils
.
mapToJson
(
mainJobs
).
spaces2
)
mainJobsWriter
.
close
()
logger
.
info
(
"done calculating dependencies"
)
}
...
...
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/WriteDependenciesTest.scala
View file @
40b3b42d
...
...
@@ -15,16 +15,14 @@
package
nl.lumc.sasc.biopet.core
import
java.io.File
import
java.nio.file.Files
import
com.google.common.io.Files
import
nl.lumc.sasc.biopet.utils.ConfigUtils
import
org.broadinstitute.gatk.queue.function.QFunction
import
org.scalatest.Matchers
import
org.scalatest.testng.TestNGSuite
import
org.testng.annotations.Test
import
scala.io.Source
/**
* Created by pjvanthof on 09/05/16.
*/
...
...
@@ -42,11 +40,14 @@ class WriteDependenciesTest extends TestNGSuite with Matchers {
@Test
def
testDeps
:
Unit
=
{
val
outputFile
=
File
.
createTempFile
(
"deps."
,
".json"
)
val
tempDir
=
Files
.
createTempDir
()
tempDir
.
deleteOnExit
()
val
prefix
=
"test"
val
outputFile
=
new
File
(
tempDir
,
s
"$prefix.deps.json"
)
outputFile
.
deleteOnExit
()
val
func1
=
Qfunc
(
file1
::
Nil
,
file2
::
Nil
)
val
func2
=
Qfunc
(
file2
::
Nil
,
file3
::
Nil
)
WriteDependencies
.
writeDependencies
(
func1
::
func2
::
Nil
,
outputFile
)
WriteDependencies
.
writeDependencies
(
func1
::
func2
::
Nil
,
tempDir
,
prefix
)
val
deps
=
ConfigUtils
.
fileToConfigMap
(
outputFile
)
deps
(
"jobs"
)
shouldBe
a
[
Map
[
_
,
_
]]
val
jobs
=
deps
(
"jobs"
).
asInstanceOf
[
Map
[
String
,
Map
[
String
,
Any
]]]
...
...
@@ -66,7 +67,7 @@ class WriteDependenciesTest extends TestNGSuite with Matchers {
}
object
WriteDependenciesTest
{
val
tempDir
=
Files
.
createTempDir
ectory
(
"test"
).
toFile
val
tempDir
=
Files
.
createTempDir
()
tempDir
.
deleteOnExit
()
val
file1
=
new
File
(
tempDir
,
"file1.txt"
)
val
file2
=
new
File
(
tempDir
,
"file2.txt"
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment