Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Mirrors
biopet.biopet
Commits
1dc47660
Commit
1dc47660
authored
Jul 11, 2017
by
pjvan_thof
Browse files
Adding scaladocs
parent
155cf263
Changes
3
Hide whitespace changes
Inline
Side-by-side
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/pipelinestatus/Deps.scala
View file @
1dc47660
...
...
@@ -5,10 +5,17 @@ import java.io.File
import
nl.lumc.sasc.biopet.utils.ConfigUtils
/**
* This class can store the deps.json from a pipeline that stores all jobs and files and the connections
*
* Created by pjvanthof on 24/06/2017.
*/
case
class
Deps
(
jobs
:
Map
[
String
,
Job
],
files
:
Array
[
Map
[
String
,
Any
]])
{
/**
* This method will compress the graph by combining all common job names
* @param main When set true the non main jobs will be skipped in the graph
* @return List of dependencies
*/
def
compressOnType
(
main
:
Boolean
=
false
)
:
Map
[
String
,
List
[
String
]]
=
{
(
for
((
_
,
job
)
<-
jobs
.
toSet
if
!
main
||
job
.
mainJob
)
yield
{
job
.
name
->
(
if
(
main
)
getMainDependencies
(
job
.
name
).
map
(
Job
.
compressedName
(
_
).
_1
)
...
...
@@ -17,10 +24,15 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
.
map
(
x
=>
x
.
_1
->
x
.
_2
.
flatMap
(
_
.
_2
).
toList
.
distinct
)
}
/** this will return all main dependencies */
def
getMainDeps
:
Map
[
String
,
List
[
String
]]
=
{
jobs
.
filter
(
_
.
_2
.
mainJob
).
map
(
x
=>
x
.
_1
->
getMainDependencies
(
x
.
_1
))
}
/**
* This will return for a single job the main dependencies.
* When a job depend on a non main job it will take the dependencies from that job till it finds a main dependency
*/
def
getMainDependencies
(
jobName
:
String
)
:
List
[
String
]
=
{
val
job
=
this
.
jobs
(
jobName
)
val
dependencies
=
job
.
dependsOnJobs
match
{
...
...
@@ -36,6 +48,8 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
}
object
Deps
{
/** This will read a deps.json and returns it as a [[Deps]] class */
def
readDepsFile
(
depsFile
:
File
)
:
Deps
=
{
val
deps
=
ConfigUtils
.
fileToConfigMap
(
depsFile
)
...
...
@@ -45,16 +59,4 @@ object Deps {
Deps
(
jobs
,
files
)
}
def
compressOnType
(
jobs
:
Map
[
String
,
List
[
String
]],
main
:
Boolean
=
false
)
:
Map
[
String
,
List
[
String
]]
=
{
val
set
=
for
((
job
,
deps
)
<-
jobs
.
toSet
;
dep
<-
deps
)
yield
{
(
Job
.
compressedName
(
job
).
_1
,
Job
.
compressedName
(
dep
).
_1
)
}
// This will collapse a Set[(String, String)] to a Map[String, List[String]]
set
.
groupBy
(
_
.
_1
).
map
(
x
=>
x
.
_1
->
x
.
_2
.
map
(
_
.
_2
).
toList
)
++
jobs
.
filter
(
_
.
_2
.
isEmpty
)
.
map
(
job
=>
Job
.
compressedName
(
job
.
_1
).
_1
->
Nil
)
}
}
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/pipelinestatus/Job.scala
View file @
1dc47660
...
...
@@ -9,33 +9,56 @@ import scala.concurrent.ExecutionContext.Implicits.global
import
scala.util.matching.Regex
/**
* This class can store a single job from a deps.json
*
* Created by pjvanthof on 24/06/2017.
*/
class
Job
(
val
name
:
String
,
map
:
Map
[
String
,
Any
])
{
/** When true this job was done at the moment of the deps.json creation */
def
doneAtStart
:
Boolean
=
ConfigUtils
.
any2boolean
(
map
(
"done_at_start"
))
/** If one of this files exist the job is marked as failed */
def
failFiles
:
List
[
File
]
=
ConfigUtils
.
any2fileList
(
map
(
"fail_files"
))
/** If all of this files exist the job is marked as done */
def
doneFiles
:
List
[
File
]
=
ConfigUtils
.
any2fileList
(
map
(
"done_files"
))
/** Returns a list of jobs that depends on this job */
def
outputUsedByJobs
:
List
[
String
]
=
ConfigUtils
.
any2stringList
(
map
(
"output_used_by_jobs"
))
/** Returns a list of job where this job depends on */
def
dependsOnJobs
:
List
[
String
]
=
ConfigUtils
.
any2stringList
(
map
(
"depends_on_jobs"
))
/** Location of the stdout file of this job */
def
stdoutFile
=
new
File
(
ConfigUtils
.
any2string
(
map
(
"stdout_file"
)))
/** All output files of this job */
def
outputsFiles
:
List
[
File
]
=
ConfigUtils
.
any2fileList
(
map
(
"outputs"
))
/** All input files of this job */
def
inputFiles
:
List
[
File
]
=
ConfigUtils
.
any2fileList
(
map
(
"inputs"
))
/** When true this job is marked as a main job in the graph */
def
mainJob
:
Boolean
=
ConfigUtils
.
any2boolean
(
map
(
"main_job"
))
/** When true this job is marked as a intermediate job */
def
intermediate
:
Boolean
=
ConfigUtils
.
any2boolean
(
map
(
"intermediate"
))
/** Return a [[Future[Boolean]] to check if the job is done */
def
isDone
:
Future
[
Boolean
]
=
Future
{
doneFiles
.
forall
(
_
.
exists
())
}
/** Return a [[Future[Boolean]] to check if the job is failed */
def
isFailed
:
Future
[
Boolean
]
=
Future
{
failFiles
.
exists
(
_
.
exists
())
}
/** Returns the compressed name of this job */
def
compressedName
:
(
String
,
Int
)
=
Job
.
compressedName
(
name
)
}
object
Job
{
val
numberRegex
:
Regex
=
"""(.*)_(\d*)$"""
.
r
/** This splits a job name from it's id */
def
compressedName
(
jobName
:
String
)
:
(
String
,
Int
)
=
jobName
match
{
case
Job
.
numberRegex
(
name
,
number
)
=>
(
name
,
number
.
toInt
)
}
...
...
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/pipelinestatus/PipelineStatus.scala
View file @
1dc47660
...
...
@@ -97,7 +97,7 @@ object PipelineStatus extends ToolCommand {
else
None
if
(
cmdArgs
.
pimHost
.
isDefined
)
{
require
(
pimRunId
.
isDefined
,
"Could not auto
genrate Pim run ID, please supply --pimRunId"
)
require
(
pimRunId
.
isDefined
,
"Could not auto
-
gen
e
rate Pim run ID, please supply --pimRunId"
)
logger
.
info
(
s
"Status will be pushed to ${cmdArgs.pimHost.get}/run/${pimRunId.get}"
)
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment