WriteSummaryTest.scala 13 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
package nl.lumc.sasc.biopet.core.summary

Peter van 't Hof's avatar
Peter van 't Hof committed
3
import java.io.{ PrintWriter, File }
Peter van 't Hof's avatar
Peter van 't Hof committed
4
5

import com.google.common.io.Files
Peter van 't Hof's avatar
Peter van 't Hof committed
6
7
import nl.lumc.sasc.biopet.core._
import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
Peter van 't Hof's avatar
Peter van 't Hof committed
8
import nl.lumc.sasc.biopet.utils.summary.Summary
Peter van 't Hof's avatar
Peter van 't Hof committed
9
10
import org.broadinstitute.gatk.queue.function.CommandLineFunction
import org.broadinstitute.gatk.queue.{ QScript, QSettings }
Peter van 't Hof's avatar
Peter van 't Hof committed
11
12
13
14
15
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import WriteSummaryTest._
import org.testng.annotations.Test

Peter van 't Hof's avatar
Peter van 't Hof committed
16
import scala.util.matching.Regex
Peter van 't Hof's avatar
Peter van 't Hof committed
17
18

/**
Peter van 't Hof's avatar
Peter van 't Hof committed
19
20
 * Created by pjvanthof on 15/01/16.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
21
22
class WriteSummaryTest extends TestNGSuite with Matchers {

Peter van 't Hof's avatar
Peter van 't Hof committed
23
24
  @Test
  def testWrongRoot(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
25
26
27
28
29
    intercept[IllegalArgumentException] {
      makeWriter(null)
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  /** This is a basic summary test, no matter the content this should always be true */
  def basicSummaryTest(summary: Summary,
                       name: String,
                       sampleId: Option[String] = None,
                       libId: Option[String] = None): Unit = {
    summary.getValue(sampleId, libId, name) should not be None
    summary.getValue(sampleId, libId, name, "files", "pipeline").get shouldBe a[Map[_, _]]
    summary.getValue(sampleId, libId, name, "settings").get shouldBe a[Map[_, _]]
    summary.getValue(sampleId, libId, name, "executables").get shouldBe a[Map[_, _]]

    summary.getValue("meta") should not be None
    summary.getValue("meta", "pipeline_name") shouldBe Some(name)
    summary.getValue("meta", "last_commit_hash") shouldBe Some(nl.lumc.sasc.biopet.LastCommitHash)
    summary.getValue("meta", "pipeline_version") shouldBe Some(nl.lumc.sasc.biopet.Version)
    summary.getValue("meta", "output_dir") shouldBe Some(new File(".").getAbsolutePath)
    summary.getValue("meta", "summary_creation") should not be None
  }

  def createFakeCheckSum(file: File): Unit = {
    file.getParentFile.mkdirs()
    val writer = new PrintWriter(file)
    writer.println("checksum  file")
    writer.close()
    file.deleteOnExit()
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
56
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
57
58
  def testEmpty(): Unit = {
    val qscript = makeQscript(name = "test")
Peter van 't Hof's avatar
Peter van 't Hof committed
59
60
    val writer = makeWriter(qscript)
    writer.freezeFieldValues()
Peter van 't Hof's avatar
Peter van 't Hof committed
61
62
    writer.deps shouldBe empty
    writer.run()
Peter van 't Hof's avatar
Peter van 't Hof committed
63

Peter van 't Hof's avatar
Peter van 't Hof committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test")
  }

  @Test
  def testMergeQscript(): Unit = {
    val qscript = makeQscript(name = "test")
    val qscript2 = makeQscript(name = "test2")
    qscript.addSummaryQScript(qscript2)
    val summaryWriter = new PrintWriter(qscript2.summaryFile)
    summaryWriter.println("""{ "test2": "value" }""")
    summaryWriter.close()
    val writer = makeWriter(qscript)
    writer.freezeFieldValues()
Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
80
    writer.run()

    val summary = new Summary(writer.out)
Peter van 't Hof's avatar
Peter van 't Hof committed
81
82
83
    basicSummaryTest(summary, "test")
    summary.getValue("test2") shouldBe Some("value")
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
84

Peter van 't Hof's avatar
Peter van 't Hof committed
85
86
87
88
89
90
91
92
93
94
  @Test
  def testSingleJob(): Unit = {
    val qscript = makeQscript("test")
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()
Peter van 't Hof's avatar
Peter van 't Hof committed
95

Peter van 't Hof's avatar
Peter van 't Hof committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test")
    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
  }

  @Test
  def testSingleJavaJob(): Unit = {
    val qscript = makeQscript("test")
    val writer = makeWriter(qscript)
    val summarizable = makeJavaCommand(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.add(summarizable)
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()

    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test")
    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
    summary.getValue("test", "executables", "java_command", "version") shouldBe Some("test version")
  }

  @Test
  def testVersion(): Unit = {
    val qscript = makeQscript("test")
    val writer = makeWriter(qscript)
    val summarizable = makeVersionSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.add(summarizable)
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()

    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test")
    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
    summary.getValue("test", "executables", "version_command", "version") shouldBe Some("test version")
  }

  @Test
  def testSampleLibrary(): Unit = {
    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName"))
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = Some("libName"))
    summary.getValue(Some("sampleName"), Some("libName"), "test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue(Some("sampleName"), Some("libName"), "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
  }

  @Test
  def testSample(): Unit = {
    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"))
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test", sampleId = Some("sampleName"), libId = None)
    summary.getValue(Some("sampleName"), None, "test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue(Some("sampleName"), None, "test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
174
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195

  @Test
  def testMultisampleQscript(): Unit = {
    val qscript = makeMultisampleQscript("test", multisampleConfig)
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

    val summary = new Summary(writer.out)
    basicSummaryTest(summary, "test")
    summary.getValue("test", "stats", "tool_1", "key") shouldBe Some("value")
    summary.getValue("test", "files", "tool_1", "file_1", "md5") shouldBe Some("checksum")

    summary.getValue(Some("sampleName"), Some("libName"), "test") should not be None
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
196
197
198
199
}

object WriteSummaryTest {
  def makeWriter(root: Configurable, c: Map[String, Any] = Map()) = new WriteSummary(root) {
Peter van 't Hof's avatar
Peter van 't Hof committed
200
    override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
201
202
203
204
205
206
207
208
209
210
211
    override def outputs = Seq()
    override def inputs = Seq()
    qSettings = new QSettings {
      jobName = "test"
      jobTempDir = Files.createTempDir()
      jobTempDir.deleteOnExit()
      jobPriority = Some(1)
    }
    override def absoluteCommandDirectory() {}
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
212
213
  def makeQscript(name: String,
                  settings: Map[String, Any] = Map(),
Peter van 't Hof's avatar
Peter van 't Hof committed
214
215
216
                  files: Map[String, File] = Map(),
                  c: Map[String, Any] = Map()) =
    new SummaryQScript with QScript {
Peter van 't Hof's avatar
Peter van 't Hof committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
      summaryName = name
      outputDir = new File(".").getAbsoluteFile
      override def globalConfig = new Config(c)
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
      def init(): Unit = {}
      def biopetScript(): Unit = {}
      def root: Configurable = null
    }

  def makeSampleLibraryQscript(name: String,
                               settings: Map[String, Any] = Map(),
                               files: Map[String, File] = Map(),
                               c: Map[String, Any] = Map(),
                               s: Option[String] = None,
                               l: Option[String] = None) =
    new SummaryQScript with QScript with SampleLibraryTag {
      sampleId = s
      libId = l
Peter van 't Hof's avatar
Peter van 't Hof committed
239
240
      summaryName = "test"
      outputDir = new File(".").getAbsoluteFile
Peter van 't Hof's avatar
Peter van 't Hof committed
241
      override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
242
243
244
245
246
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
Peter van 't Hof's avatar
Peter van 't Hof committed
247
248
      def init(): Unit = {}
      def biopetScript(): Unit = {}
Peter van 't Hof's avatar
Peter van 't Hof committed
249
250
251
      def root: Configurable = null
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
252
253
254
255
256
257
258
  def makeMultisampleQscript(name: String,
                             c: Map[String, Any],
                             settings: Map[String, Any] = Map(),
                             files: Map[String, File] = Map()) =
    new MultiSampleQScript with QScript {
      summaryName = "test"
      outputDir = new File(".").getAbsoluteFile
Peter van 't Hof's avatar
Peter van 't Hof committed
259
      override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
      def init(): Unit = {}
      def biopetScript(): Unit = {}
      def root: Configurable = null

      class Sample(id: String) extends AbstractSample(id) {
        class Library(id: String) extends AbstractLibrary(id) {
          protected def addJobs(): Unit = {}
          def summaryFiles: Map[String, File] = files
          def summaryStats: Any = Map()
        }

        def makeLibrary(id: String): Library = new Library(id)
        protected def addJobs(): Unit = {}
        def summaryFiles: Map[String, File] = files
        def summaryStats: Any = Map()
      }

      def makeSample(id: String): Sample = new Sample(id)

      def addMultiSampleJobs(): Unit = {}
    }

  val multisampleConfig = Map("samples" -> Map("sampleName" -> Map("libraries" -> Map("libName" -> Map()))))

Peter van 't Hof's avatar
Peter van 't Hof committed
289
290
291
292
  def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable {
    def summaryFiles: Map[String, File] = files
    def summaryStats: Any = stats
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345

  def makeJavaCommand(files: Map[String, File] = Map(),
                      stats: Map[String, Any] = Map(),
                      c: Map[String, Any] = Map()) = new BiopetJavaCommandLineFunction with Summarizable with Version {
    override def globalConfig = new Config(c)
    override def configName = "java_command"
    def root: Configurable = null
    def summaryStats: Map[String, Any] = stats
    def summaryFiles: Map[String, File] = files

    def versionCommand: String = "echo test version"
    def versionRegex: Regex = """(.*)""".r
    override def getVersion = Some("test version")

    override def outputs = Seq()
    override def inputs = Seq()
    qSettings = new QSettings {
      jobName = "test"
      jobTempDir = Files.createTempDir()
      jobTempDir.deleteOnExit()
      jobPriority = Some(1)
    }
    override def absoluteCommandDirectory() {}
  }

  def makeVersionSummarizable(files: Map[String, File] = Map(),
                              stats: Map[String, Any] = Map(),
                              c: Map[String, Any] = Map()) =
    new CommandLineFunction with Configurable with Summarizable with Version {
      override def globalConfig = new Config(c)
      override def configName = "version_command"
      def root: Configurable = null

      def summaryFiles: Map[String, File] = files
      def summaryStats: Any = stats

      def versionCommand: String = "echo test version"
      def versionRegex: Regex = """(.*)""".r
      override def getVersion = Some("test version")

      def commandLine: String = ""

      override def outputs = Seq()
      override def inputs = Seq()
      qSettings = new QSettings {
        jobName = "test"
        jobTempDir = Files.createTempDir()
        jobTempDir.deleteOnExit()
        jobPriority = Some(1)
      }
      override def absoluteCommandDirectory() {}
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
346
}