Merge branch 'feature_annotation_versions' into 'develop'

Fix BIOPET-396



See merge request !465

basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala

@@ -53,7 +53,7 @@ class BastyTest extends TestNGSuite with Matchers {
       var m: Map[String, Any] = BastyTest.config
       if (sample1) m = ConfigUtils.mergeMaps(BastyTest.sample1, m)
       if (sample2) m = ConfigUtils.mergeMaps(BastyTest.sample2, m)
-      if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp" -> "test.vcf.gz"), m)
+      if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp_vcf" -> "test.vcf.gz"), m)
       ConfigUtils.mergeMaps(Map(
         "multisample_variantcalling" -> multisampleCalling,
         "single_sample_variantcalling" -> sampleCalling,

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/Reference.scala

@@ -18,6 +18,7 @@ import java.io.File
 
 import htsjdk.samtools.reference.IndexedFastaSequenceFile
 import nl.lumc.sasc.biopet.core.summary.{ Summarizable, SummaryQScript }
+import nl.lumc.sasc.biopet.utils.{ LazyCheck, BamUtils, ConfigUtils, FastaUtils, Logging }
 import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
 import nl.lumc.sasc.biopet.utils.{ ConfigUtils, FastaUtils, Logging }
 
@@ -56,6 +57,12 @@ trait Reference extends Configurable {
     referenceConfigPath ::: super.subPath
   }
 
+  lazy val geneAnnotationVersion: Option[String] = config("gene_annotation_name")
+  lazy val geneAnnotationSubPath = geneAnnotationVersion.map(x => List("gene_annotations", x)).getOrElse(Nil)
+  lazy val dbsnpVersion: Option[Int] = config("dbsnp_version")
+  lazy val dbsnpSubPath: List[String] = dbsnpVersion.map(x => List("dbsnp_annotations", x.toString)).getOrElse(Nil)
+  def dbsnpVcfFile: Option[File] = config("dbsnp_vcf", extraSubPath = dbsnpSubPath)
+
   /** Returns the reference config path */
   def referenceConfigPath = {
     List("references", referenceSpecies, referenceName)

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/annotations/Annotations.scala

@@ -14,7 +14,7 @@
  */
 package nl.lumc.sasc.biopet.core.annotations
 
-import nl.lumc.sasc.biopet.core.BiopetQScript
+import nl.lumc.sasc.biopet.core.{ BiopetQScript, Reference }
 import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
 import nl.lumc.sasc.biopet.utils.LazyCheck
 import org.broadinstitute.gatk.queue.QScript
@@ -22,38 +22,38 @@ import org.broadinstitute.gatk.queue.QScript
 /**
  * Created by pjvan_thof on 1/12/16.
  */
-trait AnnotationGtf extends BiopetQScript { qscript: QScript =>
+trait AnnotationGtf extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val annotationGtf: File = {
-    val file: File = config("annotation_gtf", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_gtf_md5", freeVar = true))
+    val file: File = config("annotation_gtf", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_gtf_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   }
 }
-trait AnnotationGff extends BiopetQScript { qscript: QScript =>
+trait AnnotationGff extends BiopetQScript with Reference { qscript: QScript =>
   /** GFF reference file in GFF3 format */
   lazy val annotationGff: File = {
-    val file: File = config("annotation_gff", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_gff_md5", freeVar = true))
+    val file: File = config("annotation_gff", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_gff_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   }
 }
 
-trait AnnotationRefFlat extends BiopetQScript { qscript: QScript =>
+trait AnnotationRefFlat extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val annotationRefFlat = new LazyCheck({
-    val file: File = config("annotation_refflat", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_refflat_md5", freeVar = true))
+    val file: File = config("annotation_refflat", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_refflat_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   })
 }
 
-trait RibosomalRefFlat extends BiopetQScript { qscript: QScript =>
+trait RibosomalRefFlat extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val ribosomalRefFlat = new LazyCheck({
-    val file: Option[File] = config("ribosome_refflat", freeVar = true)
+    val file: Option[File] = config("ribosome_refflat", freeVar = true, extraSubPath = geneAnnotationSubPath)
     file match {
-      case Some(f) => inputFiles :+ InputFile(f, config("ribosome_refflat_md5", freeVar = true))
+      case Some(f) => inputFiles :+ InputFile(f, config("ribosome_refflat_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
       case _       =>
     }
     file

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala

@@ -14,20 +14,20 @@
  */
 package nl.lumc.sasc.biopet.core
 
+import java.io.File
 import java.nio.file.Paths
 
 import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
-import nl.lumc.sasc.biopet.utils.config.{ Configurable, Config }
+import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
 import org.scalatest.Matchers
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.testng.TestNGSuite
-import org.mockito.Mockito._
 import org.testng.annotations.Test
 
 /**
  * Created by pjvan_thof on 12/30/15.
  */
-class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar {
+class ReferenceTest extends TestNGSuite with Matchers {
 
   import ReferenceTest._
 
@@ -58,6 +58,20 @@ class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar {
     Logging.checkErrors(true)
   }
 
+  @Test
+  def testDbpsnp: Unit = {
+    val a = make(config :: Map("dbsnp_version" -> 1) :: testReferenceNoIndex :: Nil, fai = true, dict = true)
+    a.dbsnpVcfFile shouldBe Some(new File("vcf1"))
+    val b = make(config :: Map("dbsnp_version" -> 321) :: testReferenceNoIndex :: Nil, fai = true, dict = true)
+    b.dbsnpVcfFile shouldBe None
+
+    val c = make(config :: Map("dbsnp_version" -> 1) :: testReference :: Nil, fai = true, dict = true)
+    c.dbsnpVcfFile shouldBe None
+    val d = make(config :: Map("dbsnp_version" -> 2) :: testReference :: Nil, fai = true, dict = true)
+    d.dbsnpVcfFile shouldBe Some(new File("vcf2"))
+    val e = make(config :: Map("dbsnp_version" -> 3) :: testReference :: Nil, fai = true, dict = true)
+    e.dbsnpVcfFile shouldBe Some(new File("vcf3"))
+  }
 }
 
 object ReferenceTest {
@@ -72,13 +86,20 @@ object ReferenceTest {
     "references" -> Map(
       "test_species" -> Map(
         "test_genome" -> Map(
-          "reference_fasta" -> resourcePath("/fake_chrQ_no_index.fa")))))
+          "reference_fasta" -> resourcePath("/fake_chrQ_no_index.fa"),
+          "dbsnp_annotations" -> Map(
+            "1" -> Map("dbsnp_vcf" -> "vcf1")
+          )))))
 
   val testReference = Map(
     "references" -> Map(
       "test_species" -> Map(
         "test_genome" -> Map(
-          "reference_fasta" -> resourcePath("/fake_chrQ.fa")))))
+          "reference_fasta" -> resourcePath("/fake_chrQ.fa"),
+          "dbsnp_annotations" -> Map(
+            "2" -> Map("dbsnp_vcf" -> "vcf2"),
+            "3" -> Map("dbsnp_vcf" -> "vcf3")
+          )))))
 
   def make(configs: List[Map[String, Any]],
            r: Configurable = null,

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/annotation/AnnotationTest.scala

+package nl.lumc.sasc.biopet.core.annotation
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.BiopetQScript
+import nl.lumc.sasc.biopet.core.annotations.{ AnnotationGff, AnnotationGtf, AnnotationRefFlat }
+import nl.lumc.sasc.biopet.utils.Logging
+import nl.lumc.sasc.biopet.utils.config.Config
+import org.broadinstitute.gatk.queue.QScript
+import org.scalatest.Matchers
+import org.scalatest.testng.TestNGSuite
+import org.testng.annotations.Test
+
+/**
+ * Created by pjvanthof on 18/11/2016.
+ */
+class AnnotationTest extends TestNGSuite with Matchers {
+  @Test
+  def testAnnotationGtf: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationGtf
+    s1.annotationGtf shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationGtf
+    s2.annotationGtf shouldBe new File("no_set.gtf")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationGtf
+    s3.annotationGtf shouldBe new File("set1.gtf")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+
+  @Test
+  def testAnnotationGff: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationGff
+    s1.annotationGff shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationGff
+    s2.annotationGff shouldBe new File("no_set.gff")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationGff
+    s3.annotationGff shouldBe new File("set1.gff")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+
+  @Test
+  def testAnnotationRefFlat: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationRefFlat
+    s1.annotationRefFlat.get shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationRefFlat
+    s2.annotationRefFlat.get shouldBe new File("no_set.refFlat")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationRefFlat
+    s3.annotationRefFlat.get shouldBe new File("set1.refFlat")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+}
+
+object AnnotationTest {
+  class Script(c: Map[String, Any]) extends QScript with BiopetQScript {
+    override def globalConfig: Config = new Config(c)
+    val root = null
+
+    /** Init for pipeline */
+    def init(): Unit = ???
+
+    /** Pipeline itself */
+    def biopetScript(): Unit = ???
+  }
+
+  val config: Map[String, Any] = Map("references" -> Map("s1" -> Map("g1" -> Map(
+    "annotation_gtf" -> "no_set.gtf",
+    "annotation_gff" -> "no_set.gff",
+    "annotation_refflat" -> "no_set.refFlat",
+    "gene_annotations" -> Map("set1" -> Map(
+      "annotation_gtf" -> "set1.gtf",
+      "annotation_gff" -> "set1.gff",
+      "annotation_refflat" -> "set1.refFlat"
+    ))
+  ))))
+}
\ No newline at end of file

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/GffRead.scala

+package nl.lumc.sasc.biopet.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Version }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by pjvanthof on 20/06/16.
+ */
+class GffRead(val root: Configurable) extends BiopetCommandLineFunction {
+
+  executable = config("exe", default = "gffread", freeVar = false)
+
+  @Input
+  var input: File = _
+
+  @Output
+  var output: File = _
+
+  var T: Boolean = config("T", default = false, freeVar = false)
+
+  def cmdLine = executable +
+    (if (inputAsStdin) "" else required(input)) +
+    (if (outputAsStsout) "" else required("-o", output)) +
+    conditional(T, "-T")
+}

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Sed.scala

+package nl.lumc.sasc.biopet.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Version }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+import scala.util.matching.Regex
+
+/**
+ * Created by pjvanthof on 18/05/16.
+ */
+class Sed(val root: Configurable) extends BiopetCommandLineFunction with Version {
+  executable = config("exe", default = "sed", freeVar = false)
+
+  /** Command to get version of executable */
+  override def versionCommand: String = executable + " --version"
+
+  /** Regex to get version from version command output */
+  override def versionRegex: Regex = """sed (GNU sed) \d+.\d+.\d+""".r
+
+  @Input(required = false)
+  var inputFile: File = _
+
+  @Output
+  var outputFile: File = _
+
+  var expressions: List[String] = Nil
+
+  def cmdLine = executable +
+    repeat("-e", expressions) +
+    (if (inputAsStdin) "" else required(inputFile)) +
+    (if (outputAsStsout) "" else " > " + required(outputFile))
+
+}

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/VariantEffectPredictor.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
 import java.io.File
 
 import nl.lumc.sasc.biopet.core.summary.Summarizable
-import nl.lumc.sasc.biopet.utils.{ Logging, VcfUtils, tryToParseNumber }
+import nl.lumc.sasc.biopet.utils.{ LazyCheck, Logging, VcfUtils, tryToParseNumber }
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Reference, Version }
 import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
@@ -39,6 +39,16 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
   @Output(doc = "output file", required = true)
   var output: File = null
 
+  lazy val vepConfig = new LazyCheck({
+    val s: Option[String] = config("vep_config")
+    s
+  })
+
+  override def subPath = {
+    if (vepConfig.isSet) super.subPath ++ vepConfig()
+    else super.subPath
+  }
+
   def versionRegex = """version (\d*)""".r
   def versionCommand = executable + " " + vepScript + " --help"
 
@@ -106,7 +116,7 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
   var skibDbCheck: Boolean = config("skip_db_check", default = false)
 
   // Textual args
-  var vepConfig: Option[String] = config("config", freeVar = false)
+  var vepConfigArg: Option[String] = config("config", freeVar = false)
   var species: Option[String] = config("species", freeVar = false)
   var assembly: Option[String] = config("assembly")
   var format: Option[String] = config("format")
@@ -229,7 +239,7 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
       conditional(lrg, "--lrg") +
       conditional(noWholeGenome, "--no_whole_genome") +
       conditional(skibDbCheck, "--skip_db_check") +
-      optional("--config", vepConfig) +
+      optional("--config", vepConfigArg) +
       optional("--species", species) +
       optional("--assembly", assembly) +
       optional("--format", format) +

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/BaseRecalibrator.scala

@@ -149,7 +149,7 @@ class BaseRecalibrator(val root: Configurable) extends CommandLineGATK with Scat
   @Argument(fullName = "filter_bases_not_stored", shortName = "filterNoBases", doc = "Filter out reads with no stored bases (i.e. '*' where the sequence should be), instead of failing with an error", required = false, exclusiveOf = "", validation = "")
   var filter_bases_not_stored: Boolean = config("filter_bases_not_stored", default = false)
 
-  if (config.contains("dbsnp")) knownSites :+= new File(config("dbsnp").asString)
+  dbsnpVcfFile.foreach(knownSites :+= _)
 
   override def beforeGraph() {
     super.beforeGraph()

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/CombineGVCFs.scala

@@ -37,7 +37,7 @@ class CombineGVCFs(val root: Configurable) extends CommandLineGATK with ScatterG
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** One or more input gVCF files */
   @Input(fullName = "variant", shortName = "V", doc = "One or more input gVCF files", required = true, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/GenotypeGVCFs.scala

@@ -102,7 +102,7 @@ class GenotypeGVCFs(val root: Configurable) extends CommandLineGATK with Scatter
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Filter out reads with CIGAR containing the N operator, instead of failing with an error */
   @Argument(fullName = "filter_reads_with_N_cigar", shortName = "filterRNC", doc = "Filter out reads with CIGAR containing the N operator, instead of failing with an error", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/HaplotypeCaller.scala

@@ -42,7 +42,7 @@ class HaplotypeCaller(val root: Configurable) extends CommandLineGATK with Scatt
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** If specified, we will not trim down the active region from the full region (active + extension) to just the active interval for genotyping */
   @Argument(fullName = "dontTrimActiveRegions", shortName = "dontTrimActiveRegions", doc = "If specified, we will not trim down the active region from the full region (active + extension) to just the active interval for genotyping", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/RealignerTargetCreator.scala

@@ -68,7 +68,7 @@ class RealignerTargetCreator(val root: Configurable) extends CommandLineGATK wit
   @Argument(fullName = "filter_bases_not_stored", shortName = "filterNoBases", doc = "Filter out reads with no stored bases (i.e. '*' where the sequence should be), instead of failing with an error", required = false, exclusiveOf = "", validation = "")
   var filter_bases_not_stored: Boolean = config("", default = false)
 
-  if (config.contains("dbsnp")) known :+= new File(config("dbsnp").asString)
+  dbsnpVcfFile.foreach(known :+= _)
 
   override def beforeGraph() {
     super.beforeGraph()

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/UnifiedGenotyper.scala

@@ -213,7 +213,7 @@ class UnifiedGenotyper(val root: Configurable) extends CommandLineGATK with Scat
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Comparison VCF file */
   @Input(fullName = "comp", shortName = "comp", doc = "Comparison VCF file", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/VariantAnnotator.scala

@@ -37,7 +37,7 @@ class VariantAnnotator(val root: Configurable) extends CommandLineGATK with Scat
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Comparison VCF file */
   @Input(fullName = "comp", shortName = "comp", doc = "Comparison VCF file", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/VariantEval.scala

@@ -39,7 +39,7 @@ class VariantEval(val root: Configurable) extends CommandLineGATK {
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Evaluations that count calls at sites of true variation (e.g., indel calls) will use this argument as their gold standard for comparison */
   @Input(fullName = "goldStandard", shortName = "gold", doc = "Evaluations that count calls at sites of true variation (e.g., indel calls) will use this argument as their gold standard for comparison", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/hisat/Hisat2Build.scala

@@ -43,8 +43,7 @@ class Hisat2Build(val root: Configurable) extends BiopetCommandLineFunction with
   var exon: Option[File] = config("exon")
 
   executable = config("exe", default = "hisat2-build", freeVar = false)
-  def versionRegex = """.*hisat2-align-s version (.*)""".r
-  override def versionExitcode = List(0, 1)
+  def versionRegex = """.*hisat.*version (.*)""".r
   def versionCommand = executable + " --version"
 
   var bmax: Option[Int] = config("bmax")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/picard/NormalizeFasta.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
+package nl.lumc.sasc.biopet.extensions.picard
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by sajvanderzeeuw on 6-10-15.
+ */
+class NormalizeFasta(val root: Configurable) extends Picard {
+
+  javaMainClass = new picard.reference.NormalizeFasta().getClass.getName
+
+  @Input(doc = "The input fasta file", required = true)
+  var input: File = _
+
+  @Output(doc = "The output fasta file", required = true)
+  var output: File = _
+
+  val lineLength: Int = config("line_length")
+
+  val truncateSequenceNameAtWhitespace: Boolean = config("truncate_sequence_name_at_whitespace", default = false)
+
+  override def cmdLine = super.cmdLine +
+    (if (inputAsStdin) required("INPUT=", new File("/dev/stdin"), spaceSeparated = false)
+    else required("INPUT=", input, spaceSeparated = false)) +
+    (if (outputAsStsout) required("OUTPUT=", new File("/dev/stdout"), spaceSeparated = false)
+    else required("OUTPUT=", output, spaceSeparated = false)) +
+    required("LINE_LENGTH=", lineLength, spaceSeparated = false) +
+    conditional(truncateSequenceNameAtWhitespace, "TRUNCATE_SEQUENCE_NAMES_AT_WHITESPACE=TRUE")
+}

biopet-package/src/main/scala/nl/lumc/sasc/biopet/BiopetExecutableMain.scala

@@ -14,7 +14,6 @@
  */
 package nl.lumc.sasc.biopet
 
-import nl.lumc.sasc.biopet.pipelines.generateindexes.GenerateIndexes
 import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
 
 object BiopetExecutableMain extends BiopetExecutable {
@@ -38,7 +37,8 @@ object BiopetExecutableMain extends BiopetExecutable {
     nl.lumc.sasc.biopet.pipelines.shiva.ShivaVariantcalling,
     nl.lumc.sasc.biopet.pipelines.basty.Basty,
     nl.lumc.sasc.biopet.pipelines.shiva.Shiva,
-    GenerateIndexes
+    nl.lumc.sasc.biopet.pipelines.generateindexes.DownloadGenomes,
+    nl.lumc.sasc.biopet.pipelines.generateindexes.GenerateIndexes
   )
 
   def tools: List[MainCommand] = BiopetToolsExecutable.tools

biopet-tools-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/DownloadNcbiAssembly.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
+package nl.lumc.sasc.biopet.extensions.tools
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.ToolCommandFunction
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * @deprecated Use picard.util.BedToIntervalList instead
+ */
+class DownloadNcbiAssembly(val root: Configurable) extends ToolCommandFunction {
+  def toolObject = nl.lumc.sasc.biopet.tools.DownloadNcbiAssembly
+
+  @Output(doc = "Output fasta file", required = true)
+  var output: File = _
+
+  var outputReport: File = _
+
+  var assemblyId: String = null
+
+  var nameHeader: Option[String] = None
+
+  var mustHaveOne: List[String] = Nil
+  var mustNotHave: List[String] = Nil
+
+  override def defaultCoreMemory = 4.0
+
+  override def cmdLine = super.cmdLine +
+    required("-a", assemblyId) +
+    required("--report", outputReport) +
+    required("-o", output) +
+    optional("--nameHeader", nameHeader) +
+    repeat("--mustHaveOne", mustHaveOne) +
+    repeat("--mustNotHave", mustNotHave)
+}
+

biopet-tools-package/src/main/scala/nl/lumc/sasc/biopet/BiopetToolsExecutable.scala

@@ -50,5 +50,6 @@ object BiopetToolsExecutable extends BiopetExecutable {
     nl.lumc.sasc.biopet.tools.VcfToTsv,
     nl.lumc.sasc.biopet.tools.VcfWithVcf,
     nl.lumc.sasc.biopet.tools.VepNormalizer,
-    nl.lumc.sasc.biopet.tools.WipeReads)
+    nl.lumc.sasc.biopet.tools.WipeReads,
+    nl.lumc.sasc.biopet.tools.DownloadNcbiAssembly)
 }

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/DownloadNcbiAssembly.scala

+package nl.lumc.sasc.biopet.tools
+
+import java.io.{ File, PrintWriter }
+
+import nl.lumc.sasc.biopet.utils.ToolCommand
+
+import scala.io.Source
+
+/**
+ * Created by pjvan_thof on 21-9-16.
+ */
+object DownloadNcbiAssembly extends ToolCommand {
+
+  case class Args(assemblyId: String = null,
+                  outputFile: File = null,
+                  reportFile: Option[File] = None,
+                  contigNameHeader: Option[String] = None,
+                  mustHaveOne: List[(String, String)] = List(),
+                  mustNotHave: List[(String, String)] = List()) extends AbstractArgs
+
+  class OptParser extends AbstractOptParser {
+    opt[String]('a', "assembly id") required () unbounded () valueName "<file>" action { (x, c) =>
+      c.copy(assemblyId = x)
+    } text "refseq ID from NCBI"
+    opt[File]('o', "output") required () unbounded () valueName "<file>" action { (x, c) =>
+      c.copy(outputFile = x)
+    } text "output Fasta file"
+    opt[File]("report") unbounded () valueName "<file>" action { (x, c) =>
+      c.copy(reportFile = Some(x))
+    } text "where to write report from ncbi"
+    opt[String]("nameHeader") unbounded () valueName "<string>" action { (x, c) =>
+      c.copy(contigNameHeader = Some(x))
+    } text
+      """
+        | What column to use from the NCBI report for the name of the contigs.
+        | All columns in the report can be used but this are the most common field to choose from:
+        | - 'Sequence-Name': Name of the contig within the assembly
+        | - 'UCSC-style-name': Name of the contig used by UCSC ( like hg19 )
+        | - 'RefSeq-Accn': Unique name of the contig at RefSeq (default for NCBI)""".stripMargin
+    opt[(String, String)]("mustHaveOne") unbounded () valueName "<column_name=regex>" action { (x, c) =>
+      c.copy(mustHaveOne = (x._1, x._2) :: c.mustHaveOne)
+    } text "This can be used to filter based on the NCBI report, multiple conditions can be given, at least 1 should be true"
+    opt[(String, String)]("mustNotHave") unbounded () valueName "<column_name=regex>" action { (x, c) =>
+      c.copy(mustNotHave = (x._1, x._2) :: c.mustNotHave)
+    } text "This can be used to filter based on the NCBI report, multiple conditions can be given, all should be false"
+  }
+
+  /**
+   * @param args the command line arguments
+   */
+  def main(args: Array[String]): Unit = {
+    val argsParser = new OptParser
+    val cmdargs: Args = argsParser.parse(args, Args()) getOrElse (throw new IllegalArgumentException)
+
+    logger.info(s"Reading ${cmdargs.assemblyId} from NCBI")
+    val reader = Source.fromURL(s"ftp://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/All/${cmdargs.assemblyId}.assembly.txt")
+    val assamblyReport = reader.getLines().toList
+    reader.close()