SummaryDbTest.scala 17.1 KB
Newer Older
pjvan_thof's avatar
pjvan_thof committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
15
16
17
package nl.lumc.sasc.biopet.utils.summary.db

import java.io.File
18
import java.sql.Date
19
20
21
22
23
24
25

import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations.Test

import scala.concurrent.Await
import scala.concurrent.duration.Duration
Peter van 't Hof's avatar
WIP    
Peter van 't Hof committed
26
27
import scala.concurrent.ExecutionContext.Implicits.global

Peter van 't Hof's avatar
Peter van 't Hof committed
28
29
import SummaryDb.Implicts._
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ NoLibrary, NoModule, NoSample }
30

31
/**
Peter van 't Hof's avatar
Peter van 't Hof committed
32
 * Testing voor [[SummaryDb]]
Peter van 't Hof's avatar
Peter van 't Hof committed
33
34
 * Created by pjvanthof on 14/02/2017.
 */
35
36
class SummaryDbTest extends TestNGSuite with Matchers {

37
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
38
  def testRuns(): Unit = {
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val date = new Date(System.currentTimeMillis())

    Await.result(db.getRuns(), Duration.Inf) shouldBe empty
    val runId = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    Await.result(db.getRuns(), Duration.Inf) shouldBe Seq(Schema.Run(runId, "name", "dir", "version", "hash", date))
    val runId2 = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    Await.result(db.getRuns(), Duration.Inf) shouldBe Seq(Schema.Run(runId, "name", "dir", "version", "hash", date), Schema.Run(runId2, "name", "dir", "version", "hash", date))

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
56
  def testSamples(): Unit = {
57
58
59
60
61
62
63
64
65
66
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val date = new Date(System.currentTimeMillis())

    val runId = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    Await.result(db.getSamples(), Duration.Inf) shouldBe empty
    val sampleId = Await.result(db.createSample("test_sample", runId), Duration.Inf)
67
    Await.result(db.createOrUpdateSample("test_sample", runId), Duration.Inf) shouldBe sampleId
68
69
70
71
    Await.result(db.getSamples(), Duration.Inf) shouldBe Seq(Schema.Sample(sampleId, "test_sample", runId, None))
    Await.result(db.getSampleName(sampleId), Duration.Inf) shouldBe Some("test_sample")
    Await.result(db.getSampleId(runId, "test_sample"), Duration.Inf) shouldBe Some(sampleId)
    Await.result(db.getSampleTags(sampleId), Duration.Inf) shouldBe None
72
73
    Await.result(db.createOrUpdateSample("test_sample", runId, Some("""{"test": "test"}""")), Duration.Inf) shouldBe sampleId
    Await.result(db.getSampleTags(sampleId), Duration.Inf) shouldBe Some(Map("test" -> "test"))
74
75
76

    val sampleId2 = Await.result(db.createSample("test_sample2", runId, Some("""{"test": "test"}""")), Duration.Inf)
    Await.result(db.getSampleTags(sampleId2), Duration.Inf) shouldBe Some(Map("test" -> "test"))
77
    Await.result(db.getSamples(), Duration.Inf) shouldBe Seq(Schema.Sample(sampleId, "test_sample", runId, Some("""{"test": "test"}""")), Schema.Sample(sampleId2, "test_sample2", runId, Some("""{"test": "test"}""")))
78
79
80
81
82

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
83
  def testLibraries(): Unit = {
84
85
86
87
88
89
90
91
92
93
94
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val date = new Date(System.currentTimeMillis())

    val runId = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    val sampleId = Await.result(db.createSample("test_sample", runId), Duration.Inf)
    Await.result(db.getLibraries(), Duration.Inf) shouldBe empty
    val libraryId = Await.result(db.createLibrary("test_lib", runId, sampleId), Duration.Inf)
95
    Await.result(db.createOrUpdateLibrary("test_lib", runId, sampleId), Duration.Inf) shouldBe libraryId
96
97
98
99
100
101
102
103
104
105
106
107
108
109

    Await.result(db.getLibraries(), Duration.Inf) shouldBe Seq(Schema.Library(libraryId, "test_lib", runId, sampleId, None))
    Await.result(db.getLibraryName(libraryId), Duration.Inf) shouldBe Some("test_lib")
    Await.result(db.getLibraryId(runId, sampleId, "test_lib"), Duration.Inf) shouldBe Some(libraryId)
    Await.result(db.getLibraryTags(sampleId), Duration.Inf) shouldBe None

    val sampleId2 = Await.result(db.createLibrary("test_lib2", runId, sampleId, Some("""{"test": "test"}""")), Duration.Inf)
    Await.result(db.getLibraryTags(sampleId2), Duration.Inf) shouldBe Some(Map("test" -> "test"))
    Await.result(db.getLibraries(), Duration.Inf) shouldBe Seq(Schema.Library(sampleId, "test_lib", runId, sampleId, None), Schema.Library(sampleId2, "test_lib2", runId, sampleId, Some("""{"test": "test"}""")))

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
110
  def testPipelines(): Unit = {
111
112
113
114
115
116
117
118
119
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val date = new Date(System.currentTimeMillis())

    val runId = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    Await.result(db.getPipelines(), Duration.Inf) shouldBe empty
Peter van 't Hof's avatar
Peter van 't Hof committed
120
    Await.result(db.getPipelineName(0), Duration.Inf) shouldBe None
121
    val pipelineId = Await.result(db.createPipeline("test", runId), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
122
    Await.result(db.getPipelineName(pipelineId), Duration.Inf) shouldBe Some("test")
123
124
125
126
127
128
129
130
131
    Await.result(db.getPipelines(), Duration.Inf) shouldBe Seq(Schema.Pipeline(pipelineId, "test", runId))
    Await.result(db.getPipelineId(runId, "test"), Duration.Inf) shouldBe Some(pipelineId)
    Await.result(db.createPipeline("test", runId), Duration.Inf) shouldBe pipelineId
    Await.result(db.getPipelines(), Duration.Inf) shouldBe Seq(Schema.Pipeline(pipelineId, "test", runId))

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
132
  def testModules(): Unit = {
133
134
135
136
137
138
139
140
141
142
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val date = new Date(System.currentTimeMillis())

    val runId = Await.result(db.createRun("name", "dir", "version", "hash", date), Duration.Inf)
    val pipelineId = Await.result(db.createPipeline("test", runId), Duration.Inf)
    Await.result(db.getModules(), Duration.Inf) shouldBe empty
Peter van 't Hof's avatar
Peter van 't Hof committed
143
    Await.result(db.getModuleName(pipelineId, 0), Duration.Inf) shouldBe None
144
145
    val moduleId = Await.result(db.createModule("test", runId, pipelineId), Duration.Inf)
    Await.result(db.getmoduleId(runId, "test", pipelineId), Duration.Inf) shouldBe Some(moduleId)
Peter van 't Hof's avatar
Peter van 't Hof committed
146
    Await.result(db.getModuleName(pipelineId, moduleId), Duration.Inf) shouldBe Some("test")
147
148
149
150
151
152
153
    Await.result(db.getModules(), Duration.Inf) shouldBe Seq(Schema.Module(pipelineId, "test", runId, pipelineId))
    Await.result(db.createModule("test", runId, pipelineId), Duration.Inf) shouldBe pipelineId
    Await.result(db.getModules(), Duration.Inf) shouldBe Seq(Schema.Module(pipelineId, "test", runId, pipelineId))

    db.close()
  }

154
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
155
  def testSettings(): Unit = {
156
157
158
159
160
161
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    Await.result(db.createOrUpdateSetting(0, 0, None, None, None, """{"content": "test" }"""), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
162
    Await.result(db.getSetting(0, 0, NoModule, NoSample, NoLibrary), Duration.Inf) shouldBe Some(Map("content" -> "test"))
163
    Await.result(db.createOrUpdateSetting(0, 0, None, None, None, """{"content": "test2" }"""), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
164
    Await.result(db.getSetting(0, 0, NoModule, NoSample, NoLibrary), Duration.Inf) shouldBe Some(Map("content" -> "test2"))
Peter van 't Hof's avatar
Peter van 't Hof committed
165
    db.close()
166
167
  }

168
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
169
  def testSettingKeys(): Unit = {
170
171
172
173
174
175
176
177
178
179
180
181
182
183
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    Await.result(db.createOrUpdateSetting(0, 0, Some(0), Some(0), Some(0),
      """
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
184
185
186
    db.getSettingKeys(0, 0, 0, 0, 0, keyValues = Map()) shouldBe Map()
    db.getSettingKeys(0, 0, 0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map("content" -> Some("test"))
    db.getSettingKeys(0, 0, 0, 0, 0, keyValues = Map("content" -> List("content2", "key"))) shouldBe Map("content" -> Some("value"))
187
188
189
190
191

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
192
  def testSettingsForSamples(): Unit = {
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val sampleId = Await.result(db.createSample("test_sample", 0), Duration.Inf)

    Await.result(db.createOrUpdateSetting(0, 0, Some(0), Some(sampleId), None,
      """
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
209
210
    db.getSettingsForSamples(0, 0, 0, keyValues = Map()) shouldBe Map(0 -> Map())
    db.getSettingsForSamples(0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map(0 -> Map("content" -> Some("test")))
211
212
213
214
215

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
216
  def testSettingsForLibraries(): Unit = {
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val sampleId = Await.result(db.createSample("test_sample", 0), Duration.Inf)
    val libraryId = Await.result(db.createLibrary("test_library", 0, sampleId), Duration.Inf)

    Await.result(db.createOrUpdateSetting(0, 0, Some(0), Some(sampleId), Some(libraryId),
      """
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
234
235
    db.getSettingsForLibraries(0, 0, 0, keyValues = Map()) shouldBe Map((0, 0) -> Map())
    db.getSettingsForLibraries(0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map((0, 0) -> Map("content" -> Some("test")))
236
237
238
239

    db.close()
  }

240
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
241
  def testStats(): Unit = {
242
243
244
245
246
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

247
248
249
250
251
252
    val runId = Await.result(db.createRun("test", "", "", "", new Date(System.currentTimeMillis())), Duration.Inf)
    val pipelineId = Await.result(db.createPipeline("test_pipeline", runId), Duration.Inf)
    val moduleId = Await.result(db.createModule("test_module", runId, pipelineId), Duration.Inf)
    val sampleId = Await.result(db.createSample("test_sample", runId), Duration.Inf)
    val libraryId = Await.result(db.createLibrary("test_library", runId, sampleId), Duration.Inf)

253
254
    Await.result(db.getStatsSize(), Duration.Inf) shouldBe 0

255
    Await.result(db.createOrUpdateStat(runId, pipelineId, None, None, None, """{"content": "test" }"""), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
256
    Await.result(db.getStat(runId, pipelineId, NoModule, NoSample, NoLibrary), Duration.Inf) shouldBe Some(Map("content" -> "test"))
257
    Await.result(db.getStatsSize(), Duration.Inf) shouldBe 1
258
    Await.result(db.createOrUpdateStat(runId, pipelineId, None, None, None, """{"content": "test2" }"""), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
259
    Await.result(db.getStat(runId, pipelineId, NoModule, NoSample, NoLibrary), Duration.Inf) shouldBe Some(Map("content" -> "test2"))
260
    Await.result(db.getStatsSize(), Duration.Inf) shouldBe 1
261
262
263

    // Test join queries
    Await.result(db.createOrUpdateStat(runId, pipelineId, Some(moduleId), Some(sampleId), Some(libraryId), """{"content": "test3" }"""), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
264
    Await.result(db.getStat(runId, "test_pipeline", "test_module", "test_sample", "test_library"), Duration.Inf) shouldBe Some(Map("content" -> "test3"))
265
    Await.result(db.getStatsSize(), Duration.Inf) shouldBe 2
266

Peter van 't Hof's avatar
Peter van 't Hof committed
267
268
269
    db.close()
  }

270
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
271
  def testStatKeys(): Unit = {
272
273
274
275
276
277
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    Await.result(db.createOrUpdateStat(0, 0, Some(0), Some(0), Some(0),
Peter van 't Hof's avatar
Peter van 't Hof committed
278
      """
279
280
281
282
283
284
285
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
286
287
288
    db.getStatKeys(0, 0, 0, 0, 0, keyValues = Map()) shouldBe Map()
    db.getStatKeys(0, 0, 0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map("content" -> Some("test"))
    db.getStatKeys(0, 0, 0, 0, 0, keyValues = Map("content" -> List("content2", "key"))) shouldBe Map("content" -> Some("value"))
289
290
291
292
293

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
294
  def testStatsForSamples(): Unit = {
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val sampleId = Await.result(db.createSample("test_sample", 0), Duration.Inf)

    Await.result(db.createOrUpdateStat(0, 0, Some(0), Some(sampleId), None,
      """
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
311
312
    db.getStatsForSamples(0, 0, 0, keyValues = Map()) shouldBe Map(0 -> Map())
    db.getStatsForSamples(0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map(0 -> Map("content" -> Some("test")))
313
314
315
316
317

    db.close()
  }

  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
318
  def testStatsForLibraries(): Unit = {
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val sampleId = Await.result(db.createSample("test_sample", 0), Duration.Inf)
    val libraryId = Await.result(db.createLibrary("test_library", 0, sampleId), Duration.Inf)

    Await.result(db.createOrUpdateStat(0, 0, Some(0), Some(sampleId), Some(libraryId),
      """
        |{
        |"content": "test",
        |"content2": {
        |  "key": "value"
        |}
        | }""".stripMargin), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
336
337
    db.getStatsForLibraries(0, 0, 0, keyValues = Map()) shouldBe Map((0, 0) -> Map())
    db.getStatsForLibraries(0, 0, 0, keyValues = Map("content" -> List("content"))) shouldBe Map((0, 0) -> Map("content" -> Some("test")))
338
339
340
341

    db.close()
  }

342
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
343
  def testFiles(): Unit = {
344
345
346
347
348
349
350
351
352
353
354
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    val runId = Await.result(db.createRun("test", "", "", "", new Date(System.currentTimeMillis())), Duration.Inf)
    val pipelineId = Await.result(db.createPipeline("test_pipeline", runId), Duration.Inf)
    val moduleId = Await.result(db.createModule("test_module", runId, pipelineId), Duration.Inf)
    val sampleId = Await.result(db.createSample("test_sample", runId), Duration.Inf)
    val libraryId = Await.result(db.createLibrary("test_library", runId, sampleId), Duration.Inf)

Peter van 't Hof's avatar
Peter van 't Hof committed
355
    Await.result(db.createOrUpdateFile(runId, pipelineId, None, None, None, "key", "path", "md5", link = false, 1), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
356
    Await.result(db.getFile(runId, pipelineId, NoModule, NoSample, NoLibrary, "key"), Duration.Inf) shouldBe Some(Schema.File(0, 0, None, None, None, "key", "path", "md5", link = false, 1))
Peter van 't Hof's avatar
Peter van 't Hof committed
357
    Await.result(db.getFiles(), Duration.Inf) shouldBe Seq(Schema.File(0, 0, None, None, None, "key", "path", "md5", link = false, 1))
Peter van 't Hof's avatar
Peter van 't Hof committed
358
    Await.result(db.createOrUpdateFile(runId, pipelineId, None, None, None, "key", "path2", "md5", link = false, 1), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
359
    Await.result(db.getFile(runId, pipelineId, NoModule, NoSample, NoLibrary, "key"), Duration.Inf) shouldBe Some(Schema.File(0, 0, None, None, None, "key", "path2", "md5", link = false, 1))
Peter van 't Hof's avatar
Peter van 't Hof committed
360
    Await.result(db.getFiles(), Duration.Inf) shouldBe Seq(Schema.File(0, 0, None, None, None, "key", "path2", "md5", link = false, 1))
361
362

    // Test join queries
Peter van 't Hof's avatar
Peter van 't Hof committed
363
    Await.result(db.createOrUpdateFile(runId, pipelineId, Some(moduleId), Some(sampleId), Some(libraryId), "key", "path3", "md5", link = false, 1), Duration.Inf)
Peter van 't Hof's avatar
Peter van 't Hof committed
364
    Await.result(db.getFile(runId, "test_pipeline", "test_module", "test_sample", "test_library", "key"), Duration.Inf) shouldBe Some(Schema.File(0, 0, Some(moduleId), Some(sampleId), Some(libraryId), "key", "path3", "md5", link = false, 1))
Peter van 't Hof's avatar
Peter van 't Hof committed
365
    Await.result(db.getFiles(), Duration.Inf) shouldBe Seq(Schema.File(0, 0, None, None, None, "key", "path2", "md5", link = false, 1), Schema.File(0, 0, Some(moduleId), Some(sampleId), Some(libraryId), "key", "path3", "md5", link = false, 1))
366
367
368
369

    db.close()
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
370
  @Test
Peter van 't Hof's avatar
Peter van 't Hof committed
371
  def testExecutable(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
372
373
374
375
376
377
378
379
380
    val dbFile = File.createTempFile("summary.", ".db")
    dbFile.deleteOnExit()
    val db = SummaryDb.openSqliteSummary(dbFile)
    db.createTables()

    Await.result(db.createOrUpdateExecutable(0, "name"), Duration.Inf)
    Await.result(db.createOrUpdateExecutable(0, "name", Some("test")), Duration.Inf)
    Await.result(db.getExecutables(Some(0)), Duration.Inf).head shouldBe Schema.Executable(0, "name", Some("test"))
    db.close()
381
382
383
  }

}