GearsSingleTest.scala 8.78 KB
Newer Older
Wai Yi Leung's avatar
Wai Yi Leung committed
1
2
3
4
5
6
7
8
9
10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Wai Yi Leung's avatar
Wai Yi Leung committed
12
13
14
15
16
17
18
19
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
package nl.lumc.sasc.biopet.pipelines.gears

import java.io.File

import com.google.common.io.Files
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.core.BiopetPipe
Peter van 't Hof's avatar
Peter van 't Hof committed
21
22
import nl.lumc.sasc.biopet.extensions.centrifuge.{ Centrifuge, CentrifugeKreport }
import nl.lumc.sasc.biopet.extensions.kraken.{ Kraken, KrakenReport }
Wai Yi Leung's avatar
Wai Yi Leung committed
23
import nl.lumc.sasc.biopet.extensions.picard.SamToFastq
24
import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsView
25
import nl.lumc.sasc.biopet.extensions.tools.KrakenReportToJson
Peter van 't Hof's avatar
Peter van 't Hof committed
26
import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
Wai Yi Leung's avatar
Wai Yi Leung committed
27
28
29
30
31
32
33
import nl.lumc.sasc.biopet.utils.config.Config
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations._

/**
34
 * Test class for [[GearsSingle]]
Wai Yi Leung's avatar
Wai Yi Leung committed
35
36
37
38
 *
 * Created by wyleung on 10/22/15.
 */

Peter van 't Hof's avatar
Peter van 't Hof committed
39
abstract class TestGearsSingle extends TestNGSuite with Matchers {
40
41
  def initPipeline(map: Map[String, Any]): GearsSingle = {
    new GearsSingle {
Sander Bollen's avatar
Sander Bollen committed
42
      override def configNamespace = "gearssingle"
Wai Yi Leung's avatar
Wai Yi Leung committed
43
44
45
46
47
48
49
50

      override def globalConfig = new Config(map)

      qSettings = new QSettings
      qSettings.runName = "test"
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
51
52
53
  def paired: Boolean = false
  def hasOutputName: Boolean = false
  def kraken: Option[Boolean] = None
54
  def centrifuge: Boolean = false
Peter van 't Hof's avatar
Peter van 't Hof committed
55
56
57
58
59
60
61
  def qiimeClosed: Boolean = false
  def qiimeOpen: Boolean = false
  def qiimeRtax: Boolean = false
  def seqCount: Boolean = false
  def downsample: Option[Int] = None

  def inputMode: Option[String] = Some("fastq")
Wai Yi Leung's avatar
Wai Yi Leung committed
62

Peter van 't Hof's avatar
Peter van 't Hof committed
63
64
  @Test
  def testGears(): Unit = {
Wai Yi Leung's avatar
Wai Yi Leung committed
65
    val map = ConfigUtils.mergeMaps(Map(
Peter van 't Hof's avatar
Peter van 't Hof committed
66
      "gears_use_qiime_rtax" -> qiimeRtax,
67
      "gears_use_centrifuge" -> centrifuge,
Peter van 't Hof's avatar
Peter van 't Hof committed
68
      "gears_use_qiime_closed" -> qiimeClosed,
Peter van 't Hof's avatar
Peter van 't Hof committed
69
      "gears_use_qiime_open" -> qiimeOpen,
Peter van 't Hof's avatar
Peter van 't Hof committed
70
      "gears_use_seq_count" -> seqCount,
Peter van 't Hof's avatar
Peter van 't Hof committed
71
72
73
74
75
      "output_dir" -> TestGearsSingle.outputDir
    ) ++
      kraken.map("gears_use_kraken" -> _) ++
      downsample.map("downsample" -> _),
      Map(TestGearsSingle.executables.toSeq: _*))
Wai Yi Leung's avatar
Wai Yi Leung committed
76

77
    val gears: GearsSingle = initPipeline(map)
Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
    gears.sampleId = Some("sampleName")
    gears.libId = Some("libName")
Wai Yi Leung's avatar
Wai Yi Leung committed
80

Peter van 't Hof's avatar
Peter van 't Hof committed
81
82
    inputMode match {
      case Some("fastq") =>
Peter van 't Hof's avatar
Peter van 't Hof committed
83
84
        gears.fastqR1 = List(TestGearsSingle.r1)
        gears.fastqR2 = if (paired) List(TestGearsSingle.r2) else Nil
Peter van 't Hof's avatar
Peter van 't Hof committed
85
86
87
      case Some("bam") => gears.bamFile = Some(TestGearsSingle.bam)
      case None        =>
      case _           => new IllegalStateException(s"$inputMode not allowed as inputMode")
Wai Yi Leung's avatar
Wai Yi Leung committed
88
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
89

Wai Yi Leung's avatar
Wai Yi Leung committed
90
91
92
    if (hasOutputName)
      gears.outputName = "test"

Peter van 't Hof's avatar
Peter van 't Hof committed
93
94
95
96
    if (inputMode.isEmpty) {
      intercept[IllegalArgumentException] {
        gears.script()
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
97
      Logging.errors.clear()
Wai Yi Leung's avatar
Wai Yi Leung committed
98
    } else {
Peter van 't Hof's avatar
Peter van 't Hof committed
99
100
101
102
103
104
105
106
107
108

      gears.script()

      if (hasOutputName) {
        gears.outputName shouldBe "test"
      } else {
        // in the following cases the filename should have been determined by the filename
        gears.outputName shouldBe (if (inputMode == Some("bam")) "bamfile" else "R1")
      }

Peter van 't Hof's avatar
Peter van 't Hof committed
109
110
      val pipesJobs = gears.functions.filter(_.isInstanceOf[BiopetPipe]).flatMap(_.asInstanceOf[BiopetPipe].pipesJobs)

Peter van 't Hof's avatar
Peter van 't Hof committed
111
      gears.summarySettings("gears_use_kraken") shouldBe kraken.getOrElse(false)
Peter van 't Hof's avatar
Peter van 't Hof committed
112
113
114
115
      gears.summarySettings("gear_use_qiime_rtax") shouldBe qiimeRtax
      gears.summarySettings("gear_use_qiime_closed") shouldBe qiimeClosed
      gears.summarySettings("gear_use_qiime_open") shouldBe qiimeOpen

Peter van 't Hof's avatar
Peter van 't Hof committed
116
      gears.krakenScript.isDefined shouldBe kraken.getOrElse(false)
117
      gears.centrifugeScript.isDefined shouldBe centrifuge
Peter van 't Hof's avatar
Peter van 't Hof committed
118
119
120
121
122
123
124
125
126
      gears.qiimeClosed.isDefined shouldBe qiimeClosed
      gears.qiimeOpen.isDefined shouldBe qiimeOpen
      gears.qiimeRatx.isDefined shouldBe qiimeRtax
      gears.seqCount.isDefined shouldBe seqCount

      // SamToFastq should have started if it was started from bam
      gears.functions.count(_.isInstanceOf[SamtoolsView]) shouldBe (if (inputMode == Some("bam")) 1 else 0)
      gears.functions.count(_.isInstanceOf[SamToFastq]) shouldBe (if (inputMode == Some("bam")) 1 else 0)

Peter van 't Hof's avatar
Peter van 't Hof committed
127
128
      gears.functions.count(_.isInstanceOf[Kraken]) shouldBe (if (kraken.getOrElse(false)) 1 else 0)
      gears.functions.count(_.isInstanceOf[KrakenReport]) shouldBe (if (kraken.getOrElse(false)) 1 else 0)
129
      gears.functions.count(_.isInstanceOf[KrakenReportToJson]) shouldBe
Peter van 't Hof's avatar
Peter van 't Hof committed
130
        ((if (kraken.getOrElse(false)) 1 else 0) + (if (centrifuge) 2 else 0))
131

132
133
      pipesJobs.count(_.isInstanceOf[Centrifuge]) shouldBe (if (centrifuge) 1 else 0)
      pipesJobs.count(_.isInstanceOf[CentrifugeKreport]) shouldBe (if (centrifuge) 2 else 0)
Wai Yi Leung's avatar
Wai Yi Leung committed
134
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
135
136
  }
}
Wai Yi Leung's avatar
Wai Yi Leung committed
137

Peter van 't Hof's avatar
Peter van 't Hof committed
138
139
140
class GearsSingleNoInputTest extends TestGearsSingle {
  override def inputMode = None
}
Peter van 't Hof's avatar
Peter van 't Hof committed
141

Peter van 't Hof's avatar
Peter van 't Hof committed
142
143
144
145
class GearsSingleDefaultTest extends TestGearsSingle
class GearsSingleKrakenTest extends TestGearsSingle {
  override def kraken = Some(true)
}
146
147
148
class GearsSingleCentrifugeTest extends TestGearsSingle {
  override def centrifuge = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
149
150
151
152
153
154
155
156
157
158
159
160
class GearsSingleQiimeClosedTest extends TestGearsSingle {
  override def qiimeClosed = true
}
class GearsSingleQiimeOpenTest extends TestGearsSingle {
  override def qiimeOpen = true
}
class GearsSingleQiimeRtaxTest extends TestGearsSingle {
  override def qiimeRtax = true
}
class GearsSingleseqCountTest extends TestGearsSingle {
  override def seqCount = true
}
Wai Yi Leung's avatar
Wai Yi Leung committed
161

Peter van 't Hof's avatar
Peter van 't Hof committed
162
163
164
165
class GearsSingleKrakenPairedTest extends TestGearsSingle {
  override def paired = true
  override def kraken = Some(true)
}
166
167
168
169
class GearsSingleCentrifugePairedTest extends TestGearsSingle {
  override def paired = true
  override def centrifuge = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
class GearsSingleQiimeClosedPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeClosed = true
}
class GearsSingleQiimeOpenPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeOpen = true
}
class GearsSingleQiimeRtaxPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeRtax = true
}
class GearsSingleseqCountPairedTest extends TestGearsSingle {
  override def paired = true
  override def seqCount = true
}
Wai Yi Leung's avatar
Wai Yi Leung committed
186

Peter van 't Hof's avatar
Peter van 't Hof committed
187
188
class GearsSingleAllTest extends TestGearsSingle {
  override def kraken = Some(true)
189
  override def centrifuge = true
Peter van 't Hof's avatar
Peter van 't Hof committed
190
191
192
193
194
195
196
  override def qiimeClosed = true
  override def qiimeOpen = true
  override def qiimeRtax = true
  override def seqCount = true
}
class GearsSingleAllPairedTest extends TestGearsSingle {
  override def kraken = Some(true)
197
  override def centrifuge = true
Peter van 't Hof's avatar
Peter van 't Hof committed
198
199
200
201
202
203
  override def qiimeClosed = true
  override def qiimeOpen = true
  override def qiimeRtax = true
  override def seqCount = true
  override def paired = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
204

Peter van 't Hof's avatar
Peter van 't Hof committed
205
206
207
class GearsSingleBamTest extends TestGearsSingle {
  override def inputMode = Some("bam")
}
Peter van 't Hof's avatar
Peter van 't Hof committed
208

Peter van 't Hof's avatar
Peter van 't Hof committed
209
210
211
212
213
214
215
216
217
class GearsSingleQiimeClosedDownsampleTest extends TestGearsSingle {
  override def paired = true
  override def qiimeClosed = true
  override def downsample = Some(10000)
}
class GearsSingleQiimeOpenDownsampleTest extends TestGearsSingle {
  override def paired = true
  override def qiimeOpen = true
  override def downsample = Some(10000)
Wai Yi Leung's avatar
Wai Yi Leung committed
218
219
}

Peter van 't Hof's avatar
Peter van 't Hof committed
220
object TestGearsSingle {
Wai Yi Leung's avatar
Wai Yi Leung committed
221
  val outputDir = Files.createTempDir()
Peter van 't Hof's avatar
Peter van 't Hof committed
222
  outputDir.deleteOnExit()
Wai Yi Leung's avatar
Wai Yi Leung committed
223
224
225
226
  new File(outputDir, "input").mkdirs()

  val r1 = new File(outputDir, "input" + File.separator + "R1.fq")
  Files.touch(r1)
227
  r1.deleteOnExit()
Wai Yi Leung's avatar
Wai Yi Leung committed
228
229
  val r2 = new File(outputDir, "input" + File.separator + "R2.fq")
  Files.touch(r2)
230
  r2.deleteOnExit()
Wai Yi Leung's avatar
Wai Yi Leung committed
231
232
  val bam = new File(outputDir, "input" + File.separator + "bamfile.bam")
  Files.touch(bam)
233
  bam.deleteOnExit()
Wai Yi Leung's avatar
Wai Yi Leung committed
234
235
236

  val executables = Map(
    "kraken" -> Map("exe" -> "test", "db" -> "test"),
237
238
    "centrifuge" -> Map("exe" -> "test", "centrifuge_index" -> "test"),
    "centrifugekreport" -> Map("exe" -> "test"),
Wai Yi Leung's avatar
Wai Yi Leung committed
239
240
    "krakenreport" -> Map("exe" -> "test", "db" -> "test"),
    "sambamba" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
241
    "samtools" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
242
243
244
    "md5sum" -> Map("exe" -> "test"),
    "assigntaxonomy" -> Map("exe" -> "test"),
    "pickclosedreferenceotus" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
245
    "pickopenreferenceotus" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
246
247
248
249
250
251
252
253
    "pickotus" -> Map("exe" -> "test"),
    "pickrepset" -> Map("exe" -> "test"),
    "splitlibrariesfastq" -> Map("exe" -> "test"),
    "flash" -> Map("exe" -> "test"),
    "fastqc" -> Map("exe" -> "test"),
    "seqtk" -> Map("exe" -> "test"),
    "sickle" -> Map("exe" -> "test"),
    "cutadapt" -> Map("exe" -> "test")
Wai Yi Leung's avatar
Wai Yi Leung committed
254
255
  )
}