Commit 1dc47660 authored by pjvan_thof's avatar pjvan_thof

Adding scaladocs

parent 155cf263
......@@ -5,10 +5,17 @@ import java.io.File
import nl.lumc.sasc.biopet.utils.ConfigUtils
/**
* This class can store the deps.json from a pipeline that stores all jobs and files and the connections
*
* Created by pjvanthof on 24/06/2017.
*/
case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
/**
* This method will compress the graph by combining all common job names
* @param main When set true the non main jobs will be skipped in the graph
* @return List of dependencies
*/
def compressOnType(main: Boolean = false): Map[String, List[String]] = {
(for ((_, job) <- jobs.toSet if !main || job.mainJob) yield {
job.name -> (if (main) getMainDependencies(job.name).map(Job.compressedName(_)._1)
......@@ -17,10 +24,15 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
.map(x => x._1 -> x._2.flatMap(_._2).toList.distinct)
}
/** this will return all main dependencies */
def getMainDeps: Map[String, List[String]] = {
jobs.filter(_._2.mainJob).map(x => x._1 -> getMainDependencies(x._1))
}
/**
* This will return for a single job the main dependencies.
* When a job depend on a non main job it will take the dependencies from that job till it finds a main dependency
*/
def getMainDependencies(jobName: String): List[String] = {
val job = this.jobs(jobName)
val dependencies = job.dependsOnJobs match {
......@@ -36,6 +48,8 @@ case class Deps(jobs: Map[String, Job], files: Array[Map[String, Any]]) {
}
object Deps {
/** This will read a deps.json and returns it as a [[Deps]] class */
def readDepsFile(depsFile: File): Deps = {
val deps = ConfigUtils.fileToConfigMap(depsFile)
......@@ -45,16 +59,4 @@ object Deps {
Deps(jobs, files)
}
def compressOnType(jobs: Map[String, List[String]],
main: Boolean = false): Map[String, List[String]] = {
val set = for ((job, deps) <- jobs.toSet; dep <- deps) yield {
(Job.compressedName(job)._1, Job.compressedName(dep)._1)
}
// This will collapse a Set[(String, String)] to a Map[String, List[String]]
set.groupBy(_._1).map(x => x._1 -> x._2.map(_._2).toList) ++ jobs
.filter(_._2.isEmpty)
.map(job => Job.compressedName(job._1)._1 -> Nil)
}
}
......@@ -9,33 +9,56 @@ import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.matching.Regex
/**
* This class can store a single job from a deps.json
*
* Created by pjvanthof on 24/06/2017.
*/
class Job(val name: String, map: Map[String, Any]) {
/** When true this job was done at the moment of the deps.json creation */
def doneAtStart: Boolean = ConfigUtils.any2boolean(map("done_at_start"))
/** If one of this files exist the job is marked as failed */
def failFiles: List[File] = ConfigUtils.any2fileList(map("fail_files"))
/** If all of this files exist the job is marked as done */
def doneFiles: List[File] = ConfigUtils.any2fileList(map("done_files"))
/** Returns a list of jobs that depends on this job */
def outputUsedByJobs: List[String] = ConfigUtils.any2stringList(map("output_used_by_jobs"))
/** Returns a list of job where this job depends on */
def dependsOnJobs: List[String] = ConfigUtils.any2stringList(map("depends_on_jobs"))
/** Location of the stdout file of this job */
def stdoutFile = new File(ConfigUtils.any2string(map("stdout_file")))
/** All output files of this job */
def outputsFiles: List[File] = ConfigUtils.any2fileList(map("outputs"))
/** All input files of this job */
def inputFiles: List[File] = ConfigUtils.any2fileList(map("inputs"))
/** When true this job is marked as a main job in the graph */
def mainJob: Boolean = ConfigUtils.any2boolean(map("main_job"))
/** When true this job is marked as a intermediate job */
def intermediate: Boolean = ConfigUtils.any2boolean(map("intermediate"))
/** Return a [[Future[Boolean]] to check if the job is done */
def isDone: Future[Boolean] = Future { doneFiles.forall(_.exists()) }
/** Return a [[Future[Boolean]] to check if the job is failed */
def isFailed: Future[Boolean] = Future { failFiles.exists(_.exists()) }
/** Returns the compressed name of this job */
def compressedName: (String, Int) = Job.compressedName(name)
}
object Job {
val numberRegex: Regex = """(.*)_(\d*)$""".r
/** This splits a job name from it's id */
def compressedName(jobName: String): (String, Int) = jobName match {
case Job.numberRegex(name, number) => (name, number.toInt)
}
......
......@@ -97,7 +97,7 @@ object PipelineStatus extends ToolCommand {
else None
if (cmdArgs.pimHost.isDefined) {
require(pimRunId.isDefined, "Could not auto genrate Pim run ID, please supply --pimRunId")
require(pimRunId.isDefined, "Could not auto-generate Pim run ID, please supply --pimRunId")
logger.info(s"Status will be pushed to ${cmdArgs.pimHost.get}/run/${pimRunId.get}")
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment