GearsSingleTest.scala 8.94 KB
Newer Older
Wai Yi Leung's avatar
Wai Yi Leung committed
1 2 3 4 5 6 7 8 9 10
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
11
 * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
Wai Yi Leung's avatar
Wai Yi Leung committed
12 13 14 15 16 17 18 19
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
package nl.lumc.sasc.biopet.pipelines.gears

import java.io.File

import com.google.common.io.Files
Peter van 't Hof's avatar
Peter van 't Hof committed
20
import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
Peter van 't Hof's avatar
Peter van 't Hof committed
21 22
import nl.lumc.sasc.biopet.extensions.centrifuge.{ Centrifuge, CentrifugeKreport }
import nl.lumc.sasc.biopet.extensions.kraken.{ Kraken, KrakenReport }
Wai Yi Leung's avatar
Wai Yi Leung committed
23
import nl.lumc.sasc.biopet.extensions.picard.SamToFastq
24
import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsView
25
import nl.lumc.sasc.biopet.extensions.tools.KrakenReportToJson
Peter van 't Hof's avatar
Peter van 't Hof committed
26
import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
Wai Yi Leung's avatar
Wai Yi Leung committed
27
import nl.lumc.sasc.biopet.utils.config.Config
28
import org.apache.commons.io.FileUtils
Wai Yi Leung's avatar
Wai Yi Leung committed
29 30 31 32 33 34
import org.broadinstitute.gatk.queue.QSettings
import org.scalatest.Matchers
import org.scalatest.testng.TestNGSuite
import org.testng.annotations._

/**
35
 * Test class for [[GearsSingle]]
Wai Yi Leung's avatar
Wai Yi Leung committed
36 37 38 39
 *
 * Created by wyleung on 10/22/15.
 */

Peter van 't Hof's avatar
Peter van 't Hof committed
40
abstract class TestGearsSingle extends TestNGSuite with Matchers {
41 42
  def initPipeline(map: Map[String, Any]): GearsSingle = {
    new GearsSingle {
43
      override def configNamespace = "gearssingle"
Wai Yi Leung's avatar
Wai Yi Leung committed
44 45 46 47 48 49 50 51

      override def globalConfig = new Config(map)

      qSettings = new QSettings
      qSettings.runName = "test"
    }
  }

Peter van 't Hof's avatar
Peter van 't Hof committed
52 53 54
  def paired: Boolean = false
  def hasOutputName: Boolean = false
  def kraken: Option[Boolean] = None
55
  def centrifuge: Boolean = false
Peter van 't Hof's avatar
Peter van 't Hof committed
56 57 58 59 60 61 62
  def qiimeClosed: Boolean = false
  def qiimeOpen: Boolean = false
  def qiimeRtax: Boolean = false
  def seqCount: Boolean = false
  def downsample: Option[Int] = None

  def inputMode: Option[String] = Some("fastq")
Wai Yi Leung's avatar
Wai Yi Leung committed
63

64 65
  private var dirs: List[File] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
66 67
  @Test
  def testGears(): Unit = {
68 69
    val outputDir = TestGearsSingle.outputDir
    dirs :+= outputDir
Wai Yi Leung's avatar
Wai Yi Leung committed
70
    val map = ConfigUtils.mergeMaps(Map(
Peter van 't Hof's avatar
Peter van 't Hof committed
71
      "gears_use_qiime_rtax" -> qiimeRtax,
72
      "gears_use_centrifuge" -> centrifuge,
Peter van 't Hof's avatar
Peter van 't Hof committed
73
      "gears_use_qiime_closed" -> qiimeClosed,
Peter van 't Hof's avatar
Peter van 't Hof committed
74
      "gears_use_qiime_open" -> qiimeOpen,
Peter van 't Hof's avatar
Peter van 't Hof committed
75
      "gears_use_seq_count" -> seqCount,
76
      "output_dir" -> outputDir
Peter van 't Hof's avatar
Peter van 't Hof committed
77 78 79 80
    ) ++
      kraken.map("gears_use_kraken" -> _) ++
      downsample.map("downsample" -> _),
      Map(TestGearsSingle.executables.toSeq: _*))
Wai Yi Leung's avatar
Wai Yi Leung committed
81

82
    val gears: GearsSingle = initPipeline(map)
Peter van 't Hof's avatar
Peter van 't Hof committed
83 84
    gears.sampleId = Some("sampleName")
    gears.libId = Some("libName")
Wai Yi Leung's avatar
Wai Yi Leung committed
85

Peter van 't Hof's avatar
Peter van 't Hof committed
86 87
    inputMode match {
      case Some("fastq") =>
Peter van 't Hof's avatar
Peter van 't Hof committed
88 89
        gears.fastqR1 = List(TestGearsSingle.r1)
        gears.fastqR2 = if (paired) List(TestGearsSingle.r2) else Nil
Peter van 't Hof's avatar
Peter van 't Hof committed
90 91 92
      case Some("bam") => gears.bamFile = Some(TestGearsSingle.bam)
      case None        =>
      case _           => new IllegalStateException(s"$inputMode not allowed as inputMode")
Wai Yi Leung's avatar
Wai Yi Leung committed
93
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
94

Wai Yi Leung's avatar
Wai Yi Leung committed
95 96 97
    if (hasOutputName)
      gears.outputName = "test"

Peter van 't Hof's avatar
Peter van 't Hof committed
98 99 100 101
    if (inputMode.isEmpty) {
      intercept[IllegalArgumentException] {
        gears.script()
      }
Peter van 't Hof's avatar
Peter van 't Hof committed
102
      Logging.errors.clear()
Wai Yi Leung's avatar
Wai Yi Leung committed
103
    } else {
Peter van 't Hof's avatar
Peter van 't Hof committed
104 105 106 107 108 109 110

      gears.script()

      if (hasOutputName) {
        gears.outputName shouldBe "test"
      } else {
        // in the following cases the filename should have been determined by the filename
111
        gears.outputName shouldBe "sampleName-libName"
Peter van 't Hof's avatar
Peter van 't Hof committed
112 113
      }

Peter van 't Hof's avatar
Peter van 't Hof committed
114 115
      val pipesJobs = gears.functions.filter(_.isInstanceOf[BiopetCommandLineFunction])
        .flatMap(_.asInstanceOf[BiopetCommandLineFunction].pipesJobs)
Peter van 't Hof's avatar
Peter van 't Hof committed
116

Peter van 't Hof's avatar
Peter van 't Hof committed
117
      gears.summarySettings("gears_use_kraken") shouldBe kraken.getOrElse(false)
Peter van 't Hof's avatar
Peter van 't Hof committed
118 119 120 121
      gears.summarySettings("gear_use_qiime_rtax") shouldBe qiimeRtax
      gears.summarySettings("gear_use_qiime_closed") shouldBe qiimeClosed
      gears.summarySettings("gear_use_qiime_open") shouldBe qiimeOpen

Peter van 't Hof's avatar
Peter van 't Hof committed
122
      gears.krakenScript.isDefined shouldBe kraken.getOrElse(false)
123
      gears.centrifugeScript.isDefined shouldBe centrifuge
Peter van 't Hof's avatar
Peter van 't Hof committed
124 125 126 127 128 129 130 131 132
      gears.qiimeClosed.isDefined shouldBe qiimeClosed
      gears.qiimeOpen.isDefined shouldBe qiimeOpen
      gears.qiimeRatx.isDefined shouldBe qiimeRtax
      gears.seqCount.isDefined shouldBe seqCount

      // SamToFastq should have started if it was started from bam
      gears.functions.count(_.isInstanceOf[SamtoolsView]) shouldBe (if (inputMode == Some("bam")) 1 else 0)
      gears.functions.count(_.isInstanceOf[SamToFastq]) shouldBe (if (inputMode == Some("bam")) 1 else 0)

Peter van 't Hof's avatar
Peter van 't Hof committed
133 134
      gears.functions.count(_.isInstanceOf[Kraken]) shouldBe (if (kraken.getOrElse(false)) 1 else 0)
      gears.functions.count(_.isInstanceOf[KrakenReport]) shouldBe (if (kraken.getOrElse(false)) 1 else 0)
135
      gears.functions.count(_.isInstanceOf[KrakenReportToJson]) shouldBe
Peter van 't Hof's avatar
Peter van 't Hof committed
136
        ((if (kraken.getOrElse(false)) 1 else 0) + (if (centrifuge) 2 else 0))
137

138 139
      pipesJobs.count(_.isInstanceOf[Centrifuge]) shouldBe (if (centrifuge) 1 else 0)
      pipesJobs.count(_.isInstanceOf[CentrifugeKreport]) shouldBe (if (centrifuge) 2 else 0)
Wai Yi Leung's avatar
Wai Yi Leung committed
140
    }
Peter van 't Hof's avatar
Peter van 't Hof committed
141
  }
142 143 144 145 146

  @AfterClass
  def removeDirs: Unit = {
    dirs.foreach(FileUtils.deleteDirectory)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
147
}
Wai Yi Leung's avatar
Wai Yi Leung committed
148

Peter van 't Hof's avatar
Peter van 't Hof committed
149 150 151
class GearsSingleNoInputTest extends TestGearsSingle {
  override def inputMode = None
}
Peter van 't Hof's avatar
Peter van 't Hof committed
152

Peter van 't Hof's avatar
Peter van 't Hof committed
153 154 155 156
class GearsSingleDefaultTest extends TestGearsSingle
class GearsSingleKrakenTest extends TestGearsSingle {
  override def kraken = Some(true)
}
157 158 159
class GearsSingleCentrifugeTest extends TestGearsSingle {
  override def centrifuge = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
160 161 162 163 164 165 166 167 168 169 170 171
class GearsSingleQiimeClosedTest extends TestGearsSingle {
  override def qiimeClosed = true
}
class GearsSingleQiimeOpenTest extends TestGearsSingle {
  override def qiimeOpen = true
}
class GearsSingleQiimeRtaxTest extends TestGearsSingle {
  override def qiimeRtax = true
}
class GearsSingleseqCountTest extends TestGearsSingle {
  override def seqCount = true
}
Wai Yi Leung's avatar
Wai Yi Leung committed
172

Peter van 't Hof's avatar
Peter van 't Hof committed
173 174 175 176
class GearsSingleKrakenPairedTest extends TestGearsSingle {
  override def paired = true
  override def kraken = Some(true)
}
177 178 179 180
class GearsSingleCentrifugePairedTest extends TestGearsSingle {
  override def paired = true
  override def centrifuge = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
class GearsSingleQiimeClosedPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeClosed = true
}
class GearsSingleQiimeOpenPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeOpen = true
}
class GearsSingleQiimeRtaxPairedTest extends TestGearsSingle {
  override def paired = true
  override def qiimeRtax = true
}
class GearsSingleseqCountPairedTest extends TestGearsSingle {
  override def paired = true
  override def seqCount = true
}
Wai Yi Leung's avatar
Wai Yi Leung committed
197

Peter van 't Hof's avatar
Peter van 't Hof committed
198 199
class GearsSingleAllTest extends TestGearsSingle {
  override def kraken = Some(true)
200
  override def centrifuge = true
Peter van 't Hof's avatar
Peter van 't Hof committed
201 202 203 204 205 206 207
  override def qiimeClosed = true
  override def qiimeOpen = true
  override def qiimeRtax = true
  override def seqCount = true
}
class GearsSingleAllPairedTest extends TestGearsSingle {
  override def kraken = Some(true)
208
  override def centrifuge = true
Peter van 't Hof's avatar
Peter van 't Hof committed
209 210 211 212 213 214
  override def qiimeClosed = true
  override def qiimeOpen = true
  override def qiimeRtax = true
  override def seqCount = true
  override def paired = true
}
Peter van 't Hof's avatar
Peter van 't Hof committed
215

Peter van 't Hof's avatar
Peter van 't Hof committed
216 217 218
class GearsSingleBamTest extends TestGearsSingle {
  override def inputMode = Some("bam")
}
Peter van 't Hof's avatar
Peter van 't Hof committed
219

Peter van 't Hof's avatar
Peter van 't Hof committed
220 221 222 223 224 225 226 227 228
class GearsSingleQiimeClosedDownsampleTest extends TestGearsSingle {
  override def paired = true
  override def qiimeClosed = true
  override def downsample = Some(10000)
}
class GearsSingleQiimeOpenDownsampleTest extends TestGearsSingle {
  override def paired = true
  override def qiimeOpen = true
  override def downsample = Some(10000)
Wai Yi Leung's avatar
Wai Yi Leung committed
229 230
}

Peter van 't Hof's avatar
Peter van 't Hof committed
231
object TestGearsSingle {
232 233 234
  def outputDir = Files.createTempDir()

  val inputDir = Files.createTempDir()
Wai Yi Leung's avatar
Wai Yi Leung committed
235

236
  val r1 = new File(inputDir, "R1.fq")
Wai Yi Leung's avatar
Wai Yi Leung committed
237
  Files.touch(r1)
238
  r1.deleteOnExit()
239
  val r2 = new File(inputDir, "R2.fq")
Wai Yi Leung's avatar
Wai Yi Leung committed
240
  Files.touch(r2)
241
  r2.deleteOnExit()
242
  val bam = new File(inputDir, "bamfile.bam")
Wai Yi Leung's avatar
Wai Yi Leung committed
243
  Files.touch(bam)
244
  bam.deleteOnExit()
Wai Yi Leung's avatar
Wai Yi Leung committed
245 246

  val executables = Map(
247
    "skip_write_dependencies" -> true,
Wai Yi Leung's avatar
Wai Yi Leung committed
248
    "kraken" -> Map("exe" -> "test", "db" -> "test"),
249 250
    "centrifuge" -> Map("exe" -> "test", "centrifuge_index" -> "test"),
    "centrifugekreport" -> Map("exe" -> "test"),
Wai Yi Leung's avatar
Wai Yi Leung committed
251 252
    "krakenreport" -> Map("exe" -> "test", "db" -> "test"),
    "sambamba" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
253
    "samtools" -> Map("exe" -> "test"),
254 255 256
    "md5sum" -> Map("exe" -> "test"),
    "assigntaxonomy" -> Map("exe" -> "test"),
    "pickclosedreferenceotus" -> Map("exe" -> "test"),
Peter van 't Hof's avatar
Peter van 't Hof committed
257
    "pickopenreferenceotus" -> Map("exe" -> "test"),
258 259 260 261 262 263 264 265
    "pickotus" -> Map("exe" -> "test"),
    "pickrepset" -> Map("exe" -> "test"),
    "splitlibrariesfastq" -> Map("exe" -> "test"),
    "flash" -> Map("exe" -> "test"),
    "fastqc" -> Map("exe" -> "test"),
    "seqtk" -> Map("exe" -> "test"),
    "sickle" -> Map("exe" -> "test"),
    "cutadapt" -> Map("exe" -> "test")
Wai Yi Leung's avatar
Wai Yi Leung committed
266 267
  )
}