PipelineStatusTest.scala 10.3 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
package nl.lumc.sasc.biopet.core

Peter van 't Hof's avatar
Peter van 't Hof committed
17
import java.io.{ File, PrintWriter }
18
19
20
21
22

import com.google.common.io.Files
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test
23
24
import PipelineStatusTest.Status
import nl.lumc.sasc.biopet.utils.IoUtils._
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import org.apache.commons.io.FileUtils
26
27

/**
Peter van 't Hof's avatar
Peter van 't Hof committed
28
29
 * Created by pjvan_thof on 10-1-17.
 */
30
31
class PipelineStatusTest extends TestNGSuite with Matchers {
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
32
  def testDefault(): Unit = {
33
    val outputDir = Files.createTempDir()
Peter van 't Hof's avatar
Peter van 't Hof committed
34
    PipelineStatusTest.writeDepsToDir(outputDir)
35
36

    PipelineStatus.main(Array("-o", outputDir.toString, "-d", outputDir.toString))
37
    checkOutput(outputDir)
Peter van 't Hof's avatar
Peter van 't Hof committed
38
39

    FileUtils.deleteDirectory(outputDir)
40
41
42
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
43
  def testDepsFileArg(): Unit = {
44
    val outputDir = Files.createTempDir()
Peter van 't Hof's avatar
Peter van 't Hof committed
45
    val depsfile = PipelineStatusTest.writeDepsToDir(outputDir)
46
47

    PipelineStatus.main(Array("-o", outputDir.toString, "-d", outputDir.toString, "--depsFile", depsfile.toString))
48
    checkOutput(outputDir)
Peter van 't Hof's avatar
Peter van 't Hof committed
49
50

    FileUtils.deleteDirectory(outputDir)
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  }

  def checkOutput(outputDir: File,
                  cat: Status.Value = Status.Pending,
                  gzip: Status.Value = Status.Pending,
                  zcat: Status.Value = Status.Pending): Unit = {
    val jobsGvFile = new File(outputDir, "jobs.gv")
    val mainJobsGvFile = new File(outputDir, "main_jobs.gv")
    val compressJobsGvFile = new File(outputDir, "compress.jobs.gv")
    val compressMainJobsGvFile = new File(outputDir, "compress.main_jobs.gv")
    jobsGvFile should exist
    mainJobsGvFile should exist
    compressJobsGvFile should exist
    compressMainJobsGvFile should exist

    val jobsGvLines = getLinesFromFile(jobsGvFile)
    require(jobsGvLines.exists(_.contains("cat_1 -> gzip_1")))
    require(jobsGvLines.exists(_.contains("gzip_1 -> zcat_1")))
    require(jobsGvLines.forall(!_.contains("cat_1 -> zcat_1")))
Peter van 't Hof's avatar
Peter van 't Hof committed
70
71
72
    require(jobsGvLines.exists(_.contains("cat_1 [style = dashed]")))
    require(jobsGvLines.forall(!_.contains("gzip_1 [style = dashed]")))
    require(jobsGvLines.forall(!_.contains("zcat_1 [style = dashed]")))
73
74
75
76
77

    val mainJobsGvLines = getLinesFromFile(mainJobsGvFile)
    require(mainJobsGvLines.exists(_.contains("cat_1 -> zcat_1")))
    require(mainJobsGvLines.forall(!_.contains("cat_1 -> gzip_1")))
    require(mainJobsGvLines.forall(!_.contains("gzip_1 -> zcat_1")))
Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
80
    require(mainJobsGvLines.exists(_.contains("cat_1 [style = dashed]")))
    require(mainJobsGvLines.forall(!_.contains("gzip_1 [style = dashed]")))
    require(mainJobsGvLines.forall(!_.contains("zcat_1 [style = dashed]")))
81
82
83
84
85

    val compressJobsGvLines = getLinesFromFile(compressJobsGvFile)
    require(compressJobsGvLines.exists(_.contains("cat -> gzip")))
    require(compressJobsGvLines.exists(_.contains("gzip -> zcat")))
    require(compressJobsGvLines.forall(!_.contains("cat -> zcat")))
Peter van 't Hof's avatar
Peter van 't Hof committed
86
87
88
    require(compressJobsGvLines.exists(_.contains("cat [style = dashed]")))
    require(compressJobsGvLines.forall(!_.contains("gzip [style = dashed]")))
    require(compressJobsGvLines.forall(!_.contains("zcat [style = dashed]")))
89
90
91
92
93

    val compressMainJobsGvLines = getLinesFromFile(compressMainJobsGvFile)
    require(compressMainJobsGvLines.exists(_.contains("cat -> zcat")))
    require(compressMainJobsGvLines.forall(!_.contains("cat -> gzip")))
    require(compressMainJobsGvLines.forall(!_.contains("gzip -> zcat")))
Peter van 't Hof's avatar
Peter van 't Hof committed
94
95
96
    require(compressMainJobsGvLines.exists(_.contains("cat [style = dashed]")))
    require(compressMainJobsGvLines.forall(!_.contains("gzip [style = dashed]")))
    require(compressMainJobsGvLines.forall(!_.contains("zcat [style = dashed]")))
97

98
99
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
100
101
102
  @Test
  def testDeps(): Unit = {
    val depsFile = File.createTempFile("deps.", ".json")
Peter van 't Hof's avatar
Peter van 't Hof committed
103
    depsFile.deleteOnExit()
Peter van 't Hof's avatar
Peter van 't Hof committed
104
105
106
107
108
109
110
111
112
113
114
    PipelineStatusTest.writeDeps(depsFile, new File("/tmp"))
    val deps = PipelineStatus.readDepsFile(depsFile)

    deps.jobs.size shouldBe 3
    deps.files.length shouldBe 5

    deps.jobs("gzip_1").stdoutFile shouldBe new File("/tmp/.file.out.gz.Gzip.out")
    deps.jobs("gzip_1").outputsFiles shouldBe List(new File("/tmp/file.out.gz"), new File("/tmp/.file.out.gz.Gzip.out"))
    deps.jobs("gzip_1").inputFiles shouldBe List(new File("/tmp/file.out"))
    deps.jobs("gzip_1").doneAtStart shouldBe false
  }
115
116
117
118
}

object PipelineStatusTest {

119
120
121
122
  object Status extends Enumeration {
    val Failed, Done, Pending = Value
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
123
  def writeDepsToDir(outputDir: File): File = {
124
125
126
    require(outputDir.exists())
    val depsFile = new File(outputDir, ".log/test.1234567890/graph/deps.json")
    depsFile.getParentFile.mkdirs()
Peter van 't Hof's avatar
Peter van 't Hof committed
127
128
129
130
131
    writeDeps(depsFile, outputDir)
    depsFile
  }

  def writeDeps(depsFile: File, outputDir: File): Unit = {
132
133
134
135
136
    val writer = new PrintWriter(depsFile)
    writer.println(defaultDeps(outputDir))
    writer.close()
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
137
  def defaultDeps(outputDir: File): String =
138
139
140
    s"""
       |{
       |  "jobs" : {
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
       |    "zcat_1" : {
       |      "fail_files" : [
       |        "$outputDir/.file.out.zcat.fail",
       |        "$outputDir/..file.out.zcat.Zcat.out.fail"
       |      ],
       |      "done_at_start" : false,
       |      "output_used_by_jobs" : [
       |
       |      ],
       |      "outputs" : [
       |        "$outputDir/file.out.zcat",
       |        "$outputDir/.file.out.zcat.Zcat.out"
       |      ],
       |      "command" : "'/bin/zcat'  '$outputDir/file.out.gz'  >  '$outputDir/file.out.zcat' ",
       |      "stdout_file" : "$outputDir/.file.out.zcat.Zcat.out",
       |      "depends_on_intermediate" : false,
       |      "fail_at_start" : false,
       |      "inputs" : [
       |        "$outputDir/file.out.gz"
       |      ],
       |      "depends_on_jobs" : [
       |        "gzip_1"
       |      ],
       |      "intermediate" : false,
       |      "done_files" : [
       |        "$outputDir/.file.out.zcat.done",
       |        "$outputDir/..file.out.zcat.Zcat.out.done"
       |      ],
       |      "main_job" : true
       |    },
171
172
173
174
175
176
177
       |    "gzip_1" : {
       |      "fail_files" : [
       |        "$outputDir/.file.out.gz.fail",
       |        "$outputDir/..file.out.gz.Gzip.out.fail"
       |      ],
       |      "done_at_start" : false,
       |      "output_used_by_jobs" : [
178
       |        "zcat_1"
179
180
181
182
183
       |      ],
       |      "outputs" : [
       |        "$outputDir/file.out.gz",
       |        "$outputDir/.file.out.gz.Gzip.out"
       |      ],
184
       |      "command" : "'/bin/gzip'  -c  '$outputDir/file.out'  >  '$outputDir/file.out.gz' ",
185
186
187
188
189
190
191
192
193
       |      "stdout_file" : "$outputDir/.file.out.gz.Gzip.out",
       |      "depends_on_intermediate" : false,
       |      "fail_at_start" : false,
       |      "inputs" : [
       |        "$outputDir/file.out"
       |      ],
       |      "depends_on_jobs" : [
       |        "cat_1"
       |      ],
Peter van 't Hof's avatar
Peter van 't Hof committed
194
       |      "intermediate" : false,
195
196
197
198
       |      "done_files" : [
       |        "$outputDir/.file.out.gz.done",
       |        "$outputDir/..file.out.gz.Gzip.out.done"
       |      ],
199
       |      "main_job" : false
200
201
202
203
204
205
206
207
208
209
210
211
212
213
       |    },
       |    "cat_1" : {
       |      "fail_files" : [
       |        "$outputDir/.file.out.fail",
       |        "$outputDir/..file.out.Cat.out.fail"
       |      ],
       |      "done_at_start" : false,
       |      "output_used_by_jobs" : [
       |        "gzip_1"
       |      ],
       |      "outputs" : [
       |        "$outputDir/file.out",
       |        "$outputDir/.file.out.Cat.out"
       |      ],
214
       |      "command" : "'/bin/cat'  'test.deps'  >  '$outputDir/file.out' ",
215
216
217
218
219
220
221
222
223
       |      "stdout_file" : "$outputDir/.file.out.Cat.out",
       |      "depends_on_intermediate" : false,
       |      "fail_at_start" : false,
       |      "inputs" : [
       |        "test.deps"
       |      ],
       |      "depends_on_jobs" : [
       |        
       |      ],
Peter van 't Hof's avatar
Peter van 't Hof committed
224
       |      "intermediate" : true,
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
       |      "done_files" : [
       |        "$outputDir/.file.out.done",
       |        "$outputDir/..file.out.Cat.out.done"
       |      ],
       |      "main_job" : true
       |    }
       |  },
       |  "files" : [
       |    {
       |      "output_jobs" : [
       |        "cat_1"
       |      ],
       |      "path" : "$outputDir/file.out",
       |      "input_jobs" : [
       |        "gzip_1"
       |      ],
       |      "exists_at_start" : false,
       |      "pipeline_input" : false,
Peter van 't Hof's avatar
Peter van 't Hof committed
243
       |      "intermediate" : true
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
       |    },
       |    {
       |      "output_jobs" : [
       |        "gzip_1"
       |      ],
       |      "path" : "$outputDir/.file.out.gz.Gzip.out",
       |      "input_jobs" : [
       |        
       |      ],
       |      "exists_at_start" : false,
       |      "pipeline_input" : false,
       |      "intermediate" : false
       |    },
       |    {
       |      "output_jobs" : [
       |        "gzip_1"
       |      ],
       |      "path" : "$outputDir/file.out.gz",
       |      "input_jobs" : [
       |        
       |      ],
       |      "exists_at_start" : false,
       |      "pipeline_input" : false,
       |      "intermediate" : false
       |    },
       |    {
       |      "output_jobs" : [
       |        
       |      ],
       |      "path" : "$outputDir/test.deps",
       |      "input_jobs" : [
       |        "cat_1"
       |      ],
       |      "exists_at_start" : false,
       |      "pipeline_input" : true,
       |      "intermediate" : false
       |    },
       |    {
       |      "output_jobs" : [
       |        "cat_1"
       |      ],
       |      "path" : "$outputDir/.file.out.Cat.out",
       |      "input_jobs" : [
       |        
       |      ],
       |      "exists_at_start" : false,
       |      "pipeline_input" : false,
       |      "intermediate" : false
       |    }
       |  ]
       |}
     """.stripMargin
}