Commit 40b3b42d authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Added a jobs file

parent 130a7376
...@@ -121,10 +121,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript => ...@@ -121,10 +121,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
} }
}) })
WriteDependencies.writeDependencies(functions, WriteDependencies.writeDependencies(functions, new File(outputDir, ".log"), qSettings.runName)
new File(outputDir, s".log/${qSettings.runName}.deps.json"),
Some(new File(outputDir, s".log/${qSettings.runName}.mainJobs.json"))
)
Logging.checkErrors() Logging.checkErrors()
logger.info("Script complete without errors") logger.info("Script complete without errors")
......
...@@ -47,9 +47,10 @@ object WriteDependencies extends Logging with Configurable { ...@@ -47,9 +47,10 @@ object WriteDependencies extends Logging with Configurable {
* This method will generate a json file where information about job and file dependencies are stored * This method will generate a json file where information about job and file dependencies are stored
* *
* @param functions This should be all functions that are given to the graph of Queue * @param functions This should be all functions that are given to the graph of Queue
* @param outputFile Json file to write dependencies to * @param outputDir
* @param prefix prefix
*/ */
def writeDependencies(functions: Seq[QFunction], outputFile: File, mainJobsFile: Option[File] = None): Unit = { def writeDependencies(functions: Seq[QFunction], outputDir: File, prefix: String): Unit = {
logger.info("Start calculating dependencies") logger.info("Start calculating dependencies")
val errorOnMissingInput: Boolean = config("error_on_missing_input", false) val errorOnMissingInput: Boolean = config("error_on_missing_input", false)
...@@ -126,17 +127,7 @@ object WriteDependencies extends Logging with Configurable { ...@@ -126,17 +127,7 @@ object WriteDependencies extends Logging with Configurable {
"fail_at_start" -> f.isFail) "fail_at_start" -> f.isFail)
}.toIterator.toMap }.toIterator.toMap
mainJobsFile.foreach { file => val outputFile = new File(outputDir, s"$prefix.deps.json")
val mainJobs = jobs.filter(_._2("main_job") == true).map {
case (name, job) =>
name -> getMainDependencies(name, jobs)
}
val writer = new PrintWriter(file)
writer.println(ConfigUtils.mapToJson(mainJobs).spaces2)
writer.close()
}
logger.info(s"Writing dependencies to: $outputFile") logger.info(s"Writing dependencies to: $outputFile")
val writer = new PrintWriter(outputFile) val writer = new PrintWriter(outputFile)
writer.println(ConfigUtils.mapToJson(Map( writer.println(ConfigUtils.mapToJson(Map(
...@@ -145,6 +136,20 @@ object WriteDependencies extends Logging with Configurable { ...@@ -145,6 +136,20 @@ object WriteDependencies extends Logging with Configurable {
)).spaces2) )).spaces2)
writer.close() writer.close()
val jobsDeps = jobs.map(x => x._1 -> (x._2("depends_on_jobs")))
val jobsWriter = new PrintWriter(new File(outputDir, s"$prefix.jobs.json"))
jobsWriter.println(ConfigUtils.mapToJson(jobsDeps).spaces2)
jobsWriter.close()
val mainJobs = jobs.filter(_._2("main_job") == true).map {
case (name, job) =>
name -> getMainDependencies(name, jobs)
}
val mainJobsWriter = new PrintWriter(new File(outputDir, s"$prefix.main_jobs.json"))
mainJobsWriter.println(ConfigUtils.mapToJson(mainJobs).spaces2)
mainJobsWriter.close()
logger.info("done calculating dependencies") logger.info("done calculating dependencies")
} }
......
...@@ -15,16 +15,14 @@ ...@@ -15,16 +15,14 @@
package nl.lumc.sasc.biopet.core package nl.lumc.sasc.biopet.core
import java.io.File import java.io.File
import java.nio.file.Files
import com.google.common.io.Files
import nl.lumc.sasc.biopet.utils.ConfigUtils import nl.lumc.sasc.biopet.utils.ConfigUtils
import org.broadinstitute.gatk.queue.function.QFunction import org.broadinstitute.gatk.queue.function.QFunction
import org.scalatest.Matchers import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test import org.testng.annotations.Test
import scala.io.Source
/** /**
* Created by pjvanthof on 09/05/16. * Created by pjvanthof on 09/05/16.
*/ */
...@@ -42,11 +40,14 @@ class WriteDependenciesTest extends TestNGSuite with Matchers { ...@@ -42,11 +40,14 @@ class WriteDependenciesTest extends TestNGSuite with Matchers {
@Test @Test
def testDeps: Unit = { def testDeps: Unit = {
val outputFile = File.createTempFile("deps.", ".json") val tempDir = Files.createTempDir()
tempDir.deleteOnExit()
val prefix = "test"
val outputFile = new File(tempDir, s"$prefix.deps.json")
outputFile.deleteOnExit() outputFile.deleteOnExit()
val func1 = Qfunc(file1 :: Nil, file2 :: Nil) val func1 = Qfunc(file1 :: Nil, file2 :: Nil)
val func2 = Qfunc(file2 :: Nil, file3 :: Nil) val func2 = Qfunc(file2 :: Nil, file3 :: Nil)
WriteDependencies.writeDependencies(func1 :: func2 :: Nil, outputFile) WriteDependencies.writeDependencies(func1 :: func2 :: Nil, tempDir, prefix)
val deps = ConfigUtils.fileToConfigMap(outputFile) val deps = ConfigUtils.fileToConfigMap(outputFile)
deps("jobs") shouldBe a[Map[_, _]] deps("jobs") shouldBe a[Map[_, _]]
val jobs = deps("jobs").asInstanceOf[Map[String, Map[String, Any]]] val jobs = deps("jobs").asInstanceOf[Map[String, Map[String, Any]]]
...@@ -66,7 +67,7 @@ class WriteDependenciesTest extends TestNGSuite with Matchers { ...@@ -66,7 +67,7 @@ class WriteDependenciesTest extends TestNGSuite with Matchers {
} }
object WriteDependenciesTest { object WriteDependenciesTest {
val tempDir = Files.createTempDirectory("test").toFile val tempDir = Files.createTempDir()
tempDir.deleteOnExit() tempDir.deleteOnExit()
val file1 = new File(tempDir, "file1.txt") val file1 = new File(tempDir, "file1.txt")
val file2 = new File(tempDir, "file2.txt") val file2 = new File(tempDir, "file2.txt")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment