Skip to content
Snippets Groups Projects
Commit 4f5b8b61 authored by Sander Bollen's avatar Sander Bollen Committed by GitHub
Browse files

Merge pull request #2 from biopet/fix-biopet-498 

Fix biopet-498
parents 2cf328c5 59d477c5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/PipelineStatus.scala
+ 1
− 2
View file @ 4f5b8b61
......@@ -97,7 +97,6 @@ object PipelineStatus extends ToolCommand {
val jobsDeps = deps.jobs.map(x => x._1 -> (x._2.dependsOnJobs match {
case l: List[_] => l.map(_.toString)
case _ => throw new IllegalStateException("Value 'depends_on_jobs' is not a list")
}))
val jobsWriter = new PrintWriter(new File(outputDir, s"jobs.json"))
jobsWriter.println(ConfigUtils.mapToJson(jobsDeps).spaces2)
......@@ -151,7 +150,7 @@ object PipelineStatus extends ToolCommand {
(compressedName(job)._1, compressedName(dep)._1)
}
// This will collapse a Set[(String, String)] to a Map[String, List[String]]
set.groupBy(_._1).map(x => x._1 -> x._2.map(_._2).toList)
set.groupBy(_._1).map(x => x._1 -> x._2.map(_._2).toList) ++ jobs.filter(_._2.isEmpty).map(job => compressedName(job._1)._1 -> Nil)
}
def compressedName(jobName: String) = jobName match {
......
biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/PipelineStatusTest.scala 0 → 100644
+ 282
− 0
View file @ 4f5b8b61
package nl.lumc.sasc.biopet.core
import java.io.{File, PrintWriter}
import com.google.common.io.Files
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test
import PipelineStatusTest.Status
import nl.lumc.sasc.biopet.utils.IoUtils._
import org.apache.commons.io.FileUtils
/**
* Created by pjvan_thof on 10-1-17.
*/
class PipelineStatusTest extends TestNGSuite with Matchers {
@Test
def testDefault(): Unit = {
val outputDir = Files.createTempDir()
PipelineStatusTest.writeDepsToDir(outputDir)
PipelineStatus.main(Array("-o", outputDir.toString, "-d", outputDir.toString))
checkOutput(outputDir)
FileUtils.deleteDirectory(outputDir)
}
@Test
def testDepsFileArg(): Unit = {
val outputDir = Files.createTempDir()
val depsfile = PipelineStatusTest.writeDepsToDir(outputDir)
PipelineStatus.main(Array("-o", outputDir.toString, "-d", outputDir.toString, "--depsFile", depsfile.toString))
checkOutput(outputDir)
FileUtils.deleteDirectory(outputDir)
}
def checkOutput(outputDir: File,
cat: Status.Value = Status.Pending,
gzip: Status.Value = Status.Pending,
zcat: Status.Value = Status.Pending): Unit = {
val jobsGvFile = new File(outputDir, "jobs.gv")
val mainJobsGvFile = new File(outputDir, "main_jobs.gv")
val compressJobsGvFile = new File(outputDir, "compress.jobs.gv")
val compressMainJobsGvFile = new File(outputDir, "compress.main_jobs.gv")
jobsGvFile should exist
mainJobsGvFile should exist
compressJobsGvFile should exist
compressMainJobsGvFile should exist
val jobsGvLines = getLinesFromFile(jobsGvFile)
require(jobsGvLines.exists(_.contains("cat_1 -> gzip_1")))
require(jobsGvLines.exists(_.contains("gzip_1 -> zcat_1")))
require(jobsGvLines.forall(!_.contains("cat_1 -> zcat_1")))
require(jobsGvLines.exists(_.contains("cat_1 [style = dashed]")))
require(jobsGvLines.forall(!_.contains("gzip_1 [style = dashed]")))
require(jobsGvLines.forall(!_.contains("zcat_1 [style = dashed]")))
val mainJobsGvLines = getLinesFromFile(mainJobsGvFile)
require(mainJobsGvLines.exists(_.contains("cat_1 -> zcat_1")))
require(mainJobsGvLines.forall(!_.contains("cat_1 -> gzip_1")))
require(mainJobsGvLines.forall(!_.contains("gzip_1 -> zcat_1")))
require(mainJobsGvLines.exists(_.contains("cat_1 [style = dashed]")))
require(mainJobsGvLines.forall(!_.contains("gzip_1 [style = dashed]")))
require(mainJobsGvLines.forall(!_.contains("zcat_1 [style = dashed]")))
val compressJobsGvLines = getLinesFromFile(compressJobsGvFile)
require(compressJobsGvLines.exists(_.contains("cat -> gzip")))
require(compressJobsGvLines.exists(_.contains("gzip -> zcat")))
require(compressJobsGvLines.forall(!_.contains("cat -> zcat")))
require(compressJobsGvLines.exists(_.contains("cat [style = dashed]")))
require(compressJobsGvLines.forall(!_.contains("gzip [style = dashed]")))
require(compressJobsGvLines.forall(!_.contains("zcat [style = dashed]")))
val compressMainJobsGvLines = getLinesFromFile(compressMainJobsGvFile)
require(compressMainJobsGvLines.exists(_.contains("cat -> zcat")))
require(compressMainJobsGvLines.forall(!_.contains("cat -> gzip")))
require(compressMainJobsGvLines.forall(!_.contains("gzip -> zcat")))
require(compressMainJobsGvLines.exists(_.contains("cat [style = dashed]")))
require(compressMainJobsGvLines.forall(!_.contains("gzip [style = dashed]")))
require(compressMainJobsGvLines.forall(!_.contains("zcat [style = dashed]")))
}
@Test
def testDeps(): Unit = {
val depsFile = File.createTempFile("deps.", ".json")
depsFile.deleteOnExit()
PipelineStatusTest.writeDeps(depsFile, new File("/tmp"))
val deps = PipelineStatus.readDepsFile(depsFile)
deps.jobs.size shouldBe 3
deps.files.length shouldBe 5
deps.jobs("gzip_1").stdoutFile shouldBe new File("/tmp/.file.out.gz.Gzip.out")
deps.jobs("gzip_1").outputsFiles shouldBe List(new File("/tmp/file.out.gz"), new File("/tmp/.file.out.gz.Gzip.out"))
deps.jobs("gzip_1").inputFiles shouldBe List(new File("/tmp/file.out"))
deps.jobs("gzip_1").doneAtStart shouldBe false
}
}
object PipelineStatusTest {
object Status extends Enumeration {
val Failed, Done, Pending = Value
}
def writeDepsToDir(outputDir: File): File = {
require(outputDir.exists())
val depsFile = new File(outputDir, ".log/test.1234567890/graph/deps.json")
depsFile.getParentFile.mkdirs()
writeDeps(depsFile, outputDir)
depsFile
}
def writeDeps(depsFile: File, outputDir: File): Unit = {
val writer = new PrintWriter(depsFile)
writer.println(defaultDeps(outputDir))
writer.close()
}
def defaultDeps(outputDir: File): String =
s"""
|{
| "jobs" : {
| "zcat_1" : {
| "fail_files" : [
| "$outputDir/.file.out.zcat.fail",
| "$outputDir/..file.out.zcat.Zcat.out.fail"
| ],
| "done_at_start" : false,
| "output_used_by_jobs" : [
|
| ],
| "outputs" : [
| "$outputDir/file.out.zcat",
| "$outputDir/.file.out.zcat.Zcat.out"
| ],
| "command" : "'/bin/zcat' '$outputDir/file.out.gz' > '$outputDir/file.out.zcat' ",
| "stdout_file" : "$outputDir/.file.out.zcat.Zcat.out",
| "depends_on_intermediate" : false,
| "fail_at_start" : false,
| "inputs" : [
| "$outputDir/file.out.gz"
| ],
| "depends_on_jobs" : [
| "gzip_1"
| ],
| "intermediate" : false,
| "done_files" : [
| "$outputDir/.file.out.zcat.done",
| "$outputDir/..file.out.zcat.Zcat.out.done"
| ],
| "main_job" : true
| },
| "gzip_1" : {
| "fail_files" : [
| "$outputDir/.file.out.gz.fail",
| "$outputDir/..file.out.gz.Gzip.out.fail"
| ],
| "done_at_start" : false,
| "output_used_by_jobs" : [
| "zcat_1"
| ],
| "outputs" : [
| "$outputDir/file.out.gz",
| "$outputDir/.file.out.gz.Gzip.out"
| ],
| "command" : "'/bin/gzip' -c '$outputDir/file.out' > '$outputDir/file.out.gz' ",
| "stdout_file" : "$outputDir/.file.out.gz.Gzip.out",
| "depends_on_intermediate" : false,
| "fail_at_start" : false,
| "inputs" : [
| "$outputDir/file.out"
| ],
| "depends_on_jobs" : [
| "cat_1"
| ],
| "intermediate" : false,
| "done_files" : [
| "$outputDir/.file.out.gz.done",
| "$outputDir/..file.out.gz.Gzip.out.done"
| ],
| "main_job" : false
| },
| "cat_1" : {
| "fail_files" : [
| "$outputDir/.file.out.fail",
| "$outputDir/..file.out.Cat.out.fail"
| ],
| "done_at_start" : false,
| "output_used_by_jobs" : [
| "gzip_1"
| ],
| "outputs" : [
| "$outputDir/file.out",
| "$outputDir/.file.out.Cat.out"
| ],
| "command" : "'/bin/cat' 'test.deps' > '$outputDir/file.out' ",
| "stdout_file" : "$outputDir/.file.out.Cat.out",
| "depends_on_intermediate" : false,
| "fail_at_start" : false,
| "inputs" : [
| "test.deps"
| ],
| "depends_on_jobs" : [
|
| ],
| "intermediate" : true,
| "done_files" : [
| "$outputDir/.file.out.done",
| "$outputDir/..file.out.Cat.out.done"
| ],
| "main_job" : true
| }
| },
| "files" : [
| {
| "output_jobs" : [
| "cat_1"
| ],
| "path" : "$outputDir/file.out",
| "input_jobs" : [
| "gzip_1"
| ],
| "exists_at_start" : false,
| "pipeline_input" : false,
| "intermediate" : true
| },
| {
| "output_jobs" : [
| "gzip_1"
| ],
| "path" : "$outputDir/.file.out.gz.Gzip.out",
| "input_jobs" : [
|
| ],
| "exists_at_start" : false,
| "pipeline_input" : false,
| "intermediate" : false
| },
| {
| "output_jobs" : [
| "gzip_1"
| ],
| "path" : "$outputDir/file.out.gz",
| "input_jobs" : [
|
| ],
| "exists_at_start" : false,
| "pipeline_input" : false,
| "intermediate" : false
| },
| {
| "output_jobs" : [
|
| ],
| "path" : "$outputDir/test.deps",
| "input_jobs" : [
| "cat_1"
| ],
| "exists_at_start" : false,
| "pipeline_input" : true,
| "intermediate" : false
| },
| {
| "output_jobs" : [
| "cat_1"
| ],
| "path" : "$outputDir/.file.out.Cat.out",
| "input_jobs" : [
|
| ],
| "exists_at_start" : false,
| "pipeline_input" : false,
| "intermediate" : false
| }
| ]
|}
""".stripMargin
}
biopet-utils/src/main/scala/nl/lumc/sasc/biopet/utils/IoUtils.scala
+ 15
− 0
View file @ 4f5b8b61
......@@ -16,6 +16,8 @@ package nl.lumc.sasc.biopet.utils
import java.io.{ File, FileInputStream, FileOutputStream, InputStream }
import scala.io.Source
/**
* This object contains generic io methods
*
......@@ -68,4 +70,17 @@ object IoUtils {
if (fname.toLowerCase.endsWith(ext)) fname.dropRight(ext.length)
else fname
}
/**
* This return the contends of a file as a List[String]
*
* @param file
* @return
*/
def getLinesFromFile(file: File): List[String] = {
val reader = Source.fromFile(file)
val lines = reader.getLines().toList
reader.close()
lines
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment