WriteSummaryTest.scala 14.6 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Peter van 't Hof's avatar
Peter van 't Hof committed
12
13
14
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
15
16
package nl.lumc.sasc.biopet.core.summary

Peter van 't Hof's avatar
Peter van 't Hof committed
17
import java.io.{ File, PrintWriter }
Peter van 't Hof's avatar
Peter van 't Hof committed
18
19

import com.google.common.io.Files
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.core._
Peter van 't Hof's avatar
Peter van 't Hof committed
21
import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
Peter van 't Hof's avatar
Peter van 't Hof committed
22
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
Peter van 't Hof's avatar
Peter van 't Hof committed
23
import org.broadinstitute.gatk.queue.function.CommandLineFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
24
import org.broadinstitute.gatk.queue.{ QScript, QSettings }
Peter van 't Hof's avatar
Peter van 't Hof committed
25
26
27
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import WriteSummaryTest._
Peter van 't Hof's avatar
Peter van 't Hof committed
28
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
Peter van 't Hof's avatar
Peter van 't Hof committed
29
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoSample }
Peter van 't Hof's avatar
Peter van 't Hof committed
30
import org.apache.commons.io.FileUtils
Peter van 't Hof's avatar
Peter van 't Hof committed
31
import org.testng.annotations.{ AfterClass, Test }
Peter van 't Hof's avatar
Peter van 't Hof committed
32

Peter van 't Hof's avatar
Peter van 't Hof committed
33
34
import scala.concurrent.Await
import scala.concurrent.duration.Duration
Peter van 't Hof's avatar
Peter van 't Hof committed
35
import scala.util.matching.Regex
Peter van 't Hof's avatar
Peter van 't Hof committed
36
37

/**
Peter van 't Hof's avatar
Peter van 't Hof committed
38
39
 * Created by pjvanthof on 15/01/16.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
40
41
class WriteSummaryTest extends TestNGSuite with Matchers {

Peter van 't Hof's avatar
Peter van 't Hof committed
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  @Test
  def testCreateFile: Unit = {
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val outputDir = new File("/tmp")
    Await.ready(WriteSummary.createFile(db, 0, 0, Some(0), Some(0), Some(0), "test", new File(outputDir, "test.tsv"), outputDir), Duration.Inf)
    val file = Await.result(db.getFile(0,0,0,0,0, "test"), Duration.Inf)
    file.map(_.path) shouldBe Some("./test.tsv")

    Await.ready(WriteSummary.createFile(db, 0, 0, Some(0), Some(0), Some(0), "test", new File("/tmp2/test.tsv"), outputDir), Duration.Inf)
    val file2 = Await.result(db.getFile(0,0,0,0,0, "test"), Duration.Inf)
    file2.map(_.path) shouldBe Some("/tmp2/test.tsv")

    db.close()
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
61
62
  @Test
  def testWrongRoot(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
63
64
65
66
67
    intercept[IllegalArgumentException] {
      makeWriter(null)
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
68
69
  private var dirs: List[File] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
70
  /** This is a basic summary test, no matter the content this should always be true */
Peter van 't Hof's avatar
Peter van 't Hof committed
71
72
73
  def basicSummaryTest(summary: SummaryDb,
                       runId: Int,
                       dir: File,
Peter van 't Hof's avatar
Peter van 't Hof committed
74
75
                       sampleId: Option[String] = None,
                       libId: Option[String] = None): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
76
77
78
79
80
    val run = Await.result(summary.getRuns(runId = Some(runId)), Duration.Inf).head
    run.commitHash shouldBe nl.lumc.sasc.biopet.LastCommitHash
    run.version shouldBe nl.lumc.sasc.biopet.Version
    run.outputDir shouldBe dir.getAbsolutePath

Peter van 't Hof's avatar
Peter van 't Hof committed
81
82
83
84
85
86
87
88
89
90
  }

  def createFakeCheckSum(file: File): Unit = {
    file.getParentFile.mkdirs()
    val writer = new PrintWriter(file)
    writer.println("checksum  file")
    writer.close()
    file.deleteOnExit()
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
91
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
92
  def testEmpty(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
93
94
95
    val dir = Files.createTempDir()
    dirs :+= dir
    val qscript = makeQscript(name = "test", dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
96
97
    val writer = makeWriter(qscript)
    writer.freezeFieldValues()
Peter van 't Hof's avatar
Peter van 't Hof committed
98
99
    writer.deps shouldBe empty
    writer.run()
Peter van 't Hof's avatar
Peter van 't Hof committed
100

Peter van 't Hof's avatar
Peter van 't Hof committed
101
102
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
103
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
104

Peter van 't Hof's avatar
Peter van 't Hof committed
105
106
  @Test
  def testSingleJob(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
107
108
109
110
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeQscript("test", dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
111
112
113
114
115
116
117
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()
Peter van 't Hof's avatar
Peter van 't Hof committed
118

Peter van 't Hof's avatar
Peter van 't Hof committed
119
120
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
121
122
    summary.getStatKeys(qscript.summaryRunId, "test", "tool_1", keyValues = Map("key" -> List("key"))) shouldBe Map("key" -> Some("value"))
    Await.result(summary.getFile(qscript.summaryRunId, "test", "tool_1", key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
123
124
125
126
  }

  @Test
  def testSingleJavaJob(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
127
128
129
130
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeQscript("test", dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
131
132
133
134
135
136
137
138
139
    val writer = makeWriter(qscript)
    val summarizable = makeJavaCommand(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.add(summarizable)
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()

Peter van 't Hof's avatar
Peter van 't Hof committed
140
141
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
142
143
    summary.getStatKeys(qscript.summaryRunId, "test", "tool_1", keyValues = Map("key" -> List("key"))) shouldBe Map("key" -> Some("value"))
    Await.result(summary.getFile(qscript.summaryRunId, "test", "tool_1", NoSample, NoLibrary, key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
144
145
146
147
  }

  @Test
  def testVersion(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
148
149
150
151
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeQscript("test", dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
152
153
154
155
156
157
158
159
160
    val writer = makeWriter(qscript)
    val summarizable = makeVersionSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.add(summarizable)
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.run()

Peter van 't Hof's avatar
Peter van 't Hof committed
161
162
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
163
164
    summary.getStatKeys(qscript.summaryRunId, "test", "tool_1", keyValues = Map("key" -> List("key"))) shouldBe Map("key" -> Some("value"))
    Await.result(summary.getFile(qscript.summaryRunId, "test", "tool_1", key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
165
166
167
168
  }

  @Test
  def testSampleLibrary(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
169
170
171
172
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), l = Some("libName"), dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
173
174
175
176
177
178
179
180
181
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

Peter van 't Hof's avatar
Peter van 't Hof committed
182
183
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
184
185
    summary.getStatKeys(qscript.summaryRunId, "test", "tool_1", "sampleName", "libName", keyValues = Map("key" -> List("key"))) shouldBe Map("key" -> Some("value"))
    Await.result(summary.getFile(qscript.summaryRunId, "test", "tool_1", "sampleName", "libName", key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
186
187
188
189
  }

  @Test
  def testSample(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
190
191
192
193
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeSampleLibraryQscript("test", s = Some("sampleName"), dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
194
195
196
197
198
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
Peter van 't Hof's avatar
Peter van 't Hof committed
199
    writer.init()
Peter van 't Hof's avatar
Peter van 't Hof committed
200
201
202
203
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

Peter van 't Hof's avatar
Peter van 't Hof committed
204
205
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
206
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
207
208
209

  @Test
  def testMultisampleQscript(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
210
211
212
213
    val dir = Files.createTempDir()
    dirs :+= dir

    val qscript = makeMultisampleQscript("test", multisampleConfig, dir = dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
214
215
216
217
218
219
220
221
222
    val writer = makeWriter(qscript)
    val summarizable = makeSummarizable(files = Map("file_1" -> new File("bla")), stats = Map("key" -> "value"))
    qscript.addSummarizable(summarizable, "tool_1")
    qscript.addSummaryJobs()
    createFakeCheckSum(SummaryQScript.md5sumCache(new File("bla")))
    writer.freezeFieldValues()
    writer.deps shouldBe empty
    writer.run()

Peter van 't Hof's avatar
Peter van 't Hof committed
223
224
    val summary = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
    basicSummaryTest(summary, qscript.summaryRunId, dir)
Peter van 't Hof's avatar
Peter van 't Hof committed
225
226
    summary.getStatKeys(qscript.summaryRunId, "test", "tool_1", keyValues = Map("key" -> List("key"))) shouldBe Map("key" -> Some("value"))
    Await.result(summary.getFile(qscript.summaryRunId, "test", "tool_1", key = "file_1"), Duration.Inf).map(_.md5) shouldBe Some("checksum")
Peter van 't Hof's avatar
Peter van 't Hof committed
227
228
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
229
230
231
232
  @AfterClass
  def removeDirs: Unit = {
    dirs.foreach(FileUtils.deleteDirectory)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
233
234
235
}

object WriteSummaryTest {
236
  def makeWriter(root: SummaryQScript, c: Map[String, Any] = Map()) = new WriteSummary(root) {
Peter van 't Hof's avatar
Peter van 't Hof committed
237
    override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
238
239
240
241
242
243
244
245
246
247
248
    override def outputs = Seq()
    override def inputs = Seq()
    qSettings = new QSettings {
      jobName = "test"
      jobTempDir = Files.createTempDir()
      jobTempDir.deleteOnExit()
      jobPriority = Some(1)
    }
    override def absoluteCommandDirectory() {}
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
249
250
  def makeQscript(name: String,
                  settings: Map[String, Any] = Map(),
Peter van 't Hof's avatar
Peter van 't Hof committed
251
                  files: Map[String, File] = Map(),
Peter van 't Hof's avatar
Peter van 't Hof committed
252
253
                  c: Map[String, Any] = Map(),
                  dir: File) =
Peter van 't Hof's avatar
Peter van 't Hof committed
254
    new SummaryQScript with QScript {
Peter van 't Hof's avatar
Peter van 't Hof committed
255
      summaryName = name
Peter van 't Hof's avatar
Peter van 't Hof committed
256
      outputDir = dir
Peter van 't Hof's avatar
Peter van 't Hof committed
257
258
259
260
261
262
263
264
      override def globalConfig = new Config(c)
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
      def init(): Unit = {}
      def biopetScript(): Unit = {}
Peter van 't Hof's avatar
Peter van 't Hof committed
265
      def parent: Configurable = null
Peter van 't Hof's avatar
Peter van 't Hof committed
266
267
268
269
270
271
272
    }

  def makeSampleLibraryQscript(name: String,
                               settings: Map[String, Any] = Map(),
                               files: Map[String, File] = Map(),
                               c: Map[String, Any] = Map(),
                               s: Option[String] = None,
Peter van 't Hof's avatar
Peter van 't Hof committed
273
274
                               l: Option[String] = None,
                               dir: File) =
Peter van 't Hof's avatar
Peter van 't Hof committed
275
276
277
    new SummaryQScript with QScript with SampleLibraryTag {
      sampleId = s
      libId = l
Peter van 't Hof's avatar
Peter van 't Hof committed
278
      summaryName = "test"
Peter van 't Hof's avatar
Peter van 't Hof committed
279
      outputDir = dir
Peter van 't Hof's avatar
Peter van 't Hof committed
280
      override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
281
282
283
284
285
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
Peter van 't Hof's avatar
Peter van 't Hof committed
286
287
      def init(): Unit = {}
      def biopetScript(): Unit = {}
Peter van 't Hof's avatar
Peter van 't Hof committed
288
      def parent: Configurable = null
Peter van 't Hof's avatar
Peter van 't Hof committed
289
290
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
291
292
293
  def makeMultisampleQscript(name: String,
                             c: Map[String, Any],
                             settings: Map[String, Any] = Map(),
Peter van 't Hof's avatar
Peter van 't Hof committed
294
295
                             files: Map[String, File] = Map(),
                             dir: File) =
Peter van 't Hof's avatar
Peter van 't Hof committed
296
297
    new MultiSampleQScript with QScript {
      summaryName = "test"
Peter van 't Hof's avatar
Peter van 't Hof committed
298
      outputDir = dir
Peter van 't Hof's avatar
Peter van 't Hof committed
299
      override def globalConfig = new Config(c + ("exe" -> "test"))
Peter van 't Hof's avatar
Peter van 't Hof committed
300
301
302
303
304
305
306
      def summarySettings: Map[String, Any] = settings
      def summaryFiles: Map[String, File] = files
      val tempFile = File.createTempFile("summary", ".json")
      tempFile.deleteOnExit()
      def summaryFile: File = tempFile
      def init(): Unit = {}
      def biopetScript(): Unit = {}
Peter van 't Hof's avatar
Peter van 't Hof committed
307
      def parent: Configurable = null
Peter van 't Hof's avatar
Peter van 't Hof committed
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328

      class Sample(id: String) extends AbstractSample(id) {
        class Library(id: String) extends AbstractLibrary(id) {
          protected def addJobs(): Unit = {}
          def summaryFiles: Map[String, File] = files
          def summaryStats: Any = Map()
        }

        def makeLibrary(id: String): Library = new Library(id)
        protected def addJobs(): Unit = {}
        def summaryFiles: Map[String, File] = files
        def summaryStats: Any = Map()
      }

      def makeSample(id: String): Sample = new Sample(id)

      def addMultiSampleJobs(): Unit = {}
    }

  val multisampleConfig = Map("samples" -> Map("sampleName" -> Map("libraries" -> Map("libName" -> Map()))))

Peter van 't Hof's avatar
Peter van 't Hof committed
329
330
331
332
  def makeSummarizable(files: Map[String, File] = Map(), stats: Map[String, Any] = Map()) = new Summarizable {
    def summaryFiles: Map[String, File] = files
    def summaryStats: Any = stats
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
333
334
335
336
337

  def makeJavaCommand(files: Map[String, File] = Map(),
                      stats: Map[String, Any] = Map(),
                      c: Map[String, Any] = Map()) = new BiopetJavaCommandLineFunction with Summarizable with Version {
    override def globalConfig = new Config(c)
Sander Bollen's avatar
Sander Bollen committed
338
    override def configNamespace = "java_command"
Peter van 't Hof's avatar
Peter van 't Hof committed
339
    def parent: Configurable = null
Peter van 't Hof's avatar
Peter van 't Hof committed
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
    def summaryStats: Map[String, Any] = stats
    def summaryFiles: Map[String, File] = files

    def versionCommand: String = "echo test version"
    def versionRegex: Regex = """(.*)""".r
    override def getVersion = Some("test version")

    override def outputs = Seq()
    override def inputs = Seq()
    qSettings = new QSettings {
      jobName = "test"
      jobTempDir = Files.createTempDir()
      jobTempDir.deleteOnExit()
      jobPriority = Some(1)
    }
    override def absoluteCommandDirectory() {}
  }

  def makeVersionSummarizable(files: Map[String, File] = Map(),
                              stats: Map[String, Any] = Map(),
                              c: Map[String, Any] = Map()) =
    new CommandLineFunction with Configurable with Summarizable with Version {
      override def globalConfig = new Config(c)
Sander Bollen's avatar
Sander Bollen committed
363
      override def configNamespace = "version_command"
Peter van 't Hof's avatar
Peter van 't Hof committed
364
      def parent: Configurable = null
Peter van 't Hof's avatar
Peter van 't Hof committed
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385

      def summaryFiles: Map[String, File] = files
      def summaryStats: Any = stats

      def versionCommand: String = "echo test version"
      def versionRegex: Regex = """(.*)""".r
      override def getVersion = Some("test version")

      def commandLine: String = ""

      override def outputs = Seq()
      override def inputs = Seq()
      qSettings = new QSettings {
        jobName = "test"
        jobTempDir = Files.createTempDir()
        jobTempDir.deleteOnExit()
        jobPriority = Some(1)
      }
      override def absoluteCommandDirectory() {}
    }

Peter van 't Hof's avatar
Peter van 't Hof committed
386
}