Commit 1dc47660 authored by pjvan_thof's avatar pjvan_thof
Browse files

Adding scaladocs

parent 155cf263
...@@ -5,10 +5,17 @@ import java.io.File ...@@ -5,10 +5,17 @@ import java.io.File
import nl.lumc.sasc.biopet.utils.ConfigUtils import nl.lumc.sasc.biopet.utils.ConfigUtils
/** /**
* This class can store the deps.json from a pipeline that stores all jobs and files and the connections
*
* Created by pjvanthof on 24/06/2017. * Created by pjvanthof on 24/06/2017.
*/ */
case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) { case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
/**
* This method will compress the graph by combining all common job names
* @param main When set true the non main jobs will be skipped in the graph
* @return List of dependencies
*/
def compressOnType(main: Boolean = false): Map[String, List[String]] = { def compressOnType(main: Boolean = false): Map[String, List[String]] = {
(for ((_, job) <- jobs.toSet if !main || job.mainJob) yield { (for ((_, job) <- jobs.toSet if !main || job.mainJob) yield {
job.name -> (if (main) getMainDependencies(job.name).map(Job.compressedName(_)._1) job.name -> (if (main) getMainDependencies(job.name).map(Job.compressedName(_)._1)
...@@ -17,10 +24,15 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) { ...@@ -17,10 +24,15 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
.map(x => x._1 -> x._2.flatMap(_._2).toList.distinct) .map(x => x._1 -> x._2.flatMap(_._2).toList.distinct)
} }
/** this will return all main dependencies */
def getMainDeps: Map[String, List[String]] = { def getMainDeps: Map[String, List[String]] = {
jobs.filter(_._2.mainJob).map(x => x._1 -> getMainDependencies(x._1)) jobs.filter(_._2.mainJob).map(x => x._1 -> getMainDependencies(x._1))
} }
/**
* This will return for a single job the main dependencies.
* When a job depend on a non main job it will take the dependencies from that job till it finds a main dependency
*/
def getMainDependencies(jobName: String): List[String] = { def getMainDependencies(jobName: String): List[String] = {
val job = this.jobs(jobName) val job = this.jobs(jobName)
val dependencies = job.dependsOnJobs match { val dependencies = job.dependsOnJobs match {
...@@ -36,6 +48,8 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) { ...@@ -36,6 +48,8 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
} }
object Deps { object Deps {
/** This will read a deps.json and returns it as a [[Deps]] class */
def readDepsFile(depsFile: File): Deps = { def readDepsFile(depsFile: File): Deps = {
val deps = ConfigUtils.fileToConfigMap(depsFile) val deps = ConfigUtils.fileToConfigMap(depsFile)
...@@ -45,16 +59,4 @@ object Deps { ...@@ -45,16 +59,4 @@ object Deps {
Deps(jobs, files) Deps(jobs, files)
} }
def compressOnType(jobs: Map[String, List[String]],
main: Boolean = false): Map[String, List[String]] = {
val set = for ((job, deps) <- jobs.toSet; dep <- deps) yield {
(Job.compressedName(job)._1, Job.compressedName(dep)._1)
}
// This will collapse a Set[(String, String)] to a Map[String, List[String]]
set.groupBy(_._1).map(x => x._1 -> x._2.map(_._2).toList) ++ jobs
.filter(_._2.isEmpty)
.map(job => Job.compressedName(job._1)._1 -> Nil)
}
} }
...@@ -9,33 +9,56 @@ import scala.concurrent.ExecutionContext.Implicits.global ...@@ -9,33 +9,56 @@ import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.matching.Regex import scala.util.matching.Regex
/** /**
* This class can store a single job from a deps.json
*
* Created by pjvanthof on 24/06/2017. * Created by pjvanthof on 24/06/2017.
*/ */
class Job(val name: String, map: Map[String, Any]) { class Job(val name: String, map: Map[String, Any]) {
/** When true this job was done at the moment of the deps.json creation */
def doneAtStart: Boolean = ConfigUtils.any2boolean(map("done_at_start")) def doneAtStart: Boolean = ConfigUtils.any2boolean(map("done_at_start"))
/** If one of this files exist the job is marked as failed */
def failFiles: List[File] = ConfigUtils.any2fileList(map("fail_files")) def failFiles: List[File] = ConfigUtils.any2fileList(map("fail_files"))
/** If all of this files exist the job is marked as done */
def doneFiles: List[File] = ConfigUtils.any2fileList(map("done_files")) def doneFiles: List[File] = ConfigUtils.any2fileList(map("done_files"))
/** Returns a list of jobs that depends on this job */
def outputUsedByJobs: List[String] = ConfigUtils.any2stringList(map("output_used_by_jobs")) def outputUsedByJobs: List[String] = ConfigUtils.any2stringList(map("output_used_by_jobs"))
/** Returns a list of job where this job depends on */
def dependsOnJobs: List[String] = ConfigUtils.any2stringList(map("depends_on_jobs")) def dependsOnJobs: List[String] = ConfigUtils.any2stringList(map("depends_on_jobs"))
/** Location of the stdout file of this job */
def stdoutFile = new File(ConfigUtils.any2string(map("stdout_file"))) def stdoutFile = new File(ConfigUtils.any2string(map("stdout_file")))
/** All output files of this job */
def outputsFiles: List[File] = ConfigUtils.any2fileList(map("outputs")) def outputsFiles: List[File] = ConfigUtils.any2fileList(map("outputs"))
/** All input files of this job */
def inputFiles: List[File] = ConfigUtils.any2fileList(map("inputs")) def inputFiles: List[File] = ConfigUtils.any2fileList(map("inputs"))
/** When true this job is marked as a main job in the graph */
def mainJob: Boolean = ConfigUtils.any2boolean(map("main_job")) def mainJob: Boolean = ConfigUtils.any2boolean(map("main_job"))
/** When true this job is marked as a intermediate job */
def intermediate: Boolean = ConfigUtils.any2boolean(map("intermediate")) def intermediate: Boolean = ConfigUtils.any2boolean(map("intermediate"))
/** Return a [[Future[Boolean]] to check if the job is done */
def isDone: Future[Boolean] = Future { doneFiles.forall(_.exists()) } def isDone: Future[Boolean] = Future { doneFiles.forall(_.exists()) }
/** Return a [[Future[Boolean]] to check if the job is failed */
def isFailed: Future[Boolean] = Future { failFiles.exists(_.exists()) } def isFailed: Future[Boolean] = Future { failFiles.exists(_.exists()) }
/** Returns the compressed name of this job */
def compressedName: (String, Int) = Job.compressedName(name) def compressedName: (String, Int) = Job.compressedName(name)
} }
object Job { object Job {
val numberRegex: Regex = """(.*)_(\d*)$""".r val numberRegex: Regex = """(.*)_(\d*)$""".r
/** This splits a job name from it's id */
def compressedName(jobName: String): (String, Int) = jobName match { def compressedName(jobName: String): (String, Int) = jobName match {
case Job.numberRegex(name, number) => (name, number.toInt) case Job.numberRegex(name, number) => (name, number.toInt)
} }
......
...@@ -97,7 +97,7 @@ object PipelineStatus extends ToolCommand { ...@@ -97,7 +97,7 @@ object PipelineStatus extends ToolCommand {
else None else None
if (cmdArgs.pimHost.isDefined) { if (cmdArgs.pimHost.isDefined) {
require(pimRunId.isDefined, "Could not auto genrate Pim run ID, please supply --pimRunId") require(pimRunId.isDefined, "Could not auto-generate Pim run ID, please supply --pimRunId")
logger.info(s"Status will be pushed to ${cmdArgs.pimHost.get}/run/${pimRunId.get}") logger.info(s"Status will be pushed to ${cmdArgs.pimHost.get}/run/${pimRunId.get}")
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment