Merge remote-tracking branch 'remotes/origin/develop' into fix-BIOPET-374

basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala

@@ -53,7 +53,7 @@ class BastyTest extends TestNGSuite with Matchers {
       var m: Map[String, Any] = BastyTest.config
       if (sample1) m = ConfigUtils.mergeMaps(BastyTest.sample1, m)
       if (sample2) m = ConfigUtils.mergeMaps(BastyTest.sample2, m)
-      if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp" -> "test.vcf.gz"), m)
+      if (dbsnp) m = ConfigUtils.mergeMaps(Map("dbsnp_vcf" -> "test.vcf.gz"), m)
       ConfigUtils.mergeMaps(Map(
         "multisample_variantcalling" -> multisampleCalling,
         "single_sample_variantcalling" -> sampleCalling,

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -111,7 +111,8 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
 
     functions.filter(_.jobOutputFile == null).foreach(f => {
       try {
-        f.jobOutputFile = new File(f.firstOutput.getAbsoluteFile.getParent, "." + f.firstOutput.getName + "." + f.getClass.getSimpleName + ".out")
+        val className = if (f.getClass.isAnonymousClass) f.getClass.getSuperclass.getSimpleName else f.getClass.getSimpleName
+        f.jobOutputFile = new File(f.firstOutput.getAbsoluteFile.getParent, "." + f.firstOutput.getName + "." + className + ".out")
       } catch {
         case e: NullPointerException => logger.warn(s"Can't generate a jobOutputFile for $f")
       }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/Reference.scala

@@ -18,6 +18,7 @@ import java.io.File
 
 import htsjdk.samtools.reference.IndexedFastaSequenceFile
 import nl.lumc.sasc.biopet.core.summary.{ Summarizable, SummaryQScript }
+import nl.lumc.sasc.biopet.utils.{ LazyCheck, BamUtils, ConfigUtils, FastaUtils, Logging }
 import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
 import nl.lumc.sasc.biopet.utils.{ ConfigUtils, FastaUtils, Logging }
 
@@ -56,6 +57,12 @@ trait Reference extends Configurable {
     referenceConfigPath ::: super.subPath
   }
 
+  lazy val geneAnnotationVersion: Option[String] = config("gene_annotation_name")
+  lazy val geneAnnotationSubPath = geneAnnotationVersion.map(x => List("gene_annotations", x)).getOrElse(Nil)
+  lazy val dbsnpVersion: Option[Int] = config("dbsnp_version")
+  lazy val dbsnpSubPath: List[String] = dbsnpVersion.map(x => List("dbsnp_annotations", x.toString)).getOrElse(Nil)
+  def dbsnpVcfFile: Option[File] = config("dbsnp_vcf", extraSubPath = dbsnpSubPath)
+
   /** Returns the reference config path */
   def referenceConfigPath = {
     List("references", referenceSpecies, referenceName)

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/annotations/Annotations.scala

@@ -14,7 +14,7 @@
  */
 package nl.lumc.sasc.biopet.core.annotations
 
-import nl.lumc.sasc.biopet.core.BiopetQScript
+import nl.lumc.sasc.biopet.core.{ BiopetQScript, Reference }
 import nl.lumc.sasc.biopet.core.BiopetQScript.InputFile
 import nl.lumc.sasc.biopet.utils.LazyCheck
 import org.broadinstitute.gatk.queue.QScript
@@ -22,38 +22,38 @@ import org.broadinstitute.gatk.queue.QScript
 /**
  * Created by pjvan_thof on 1/12/16.
  */
-trait AnnotationGtf extends BiopetQScript { qscript: QScript =>
+trait AnnotationGtf extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val annotationGtf: File = {
-    val file: File = config("annotation_gtf", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_gtf_md5", freeVar = true))
+    val file: File = config("annotation_gtf", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_gtf_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   }
 }
-trait AnnotationGff extends BiopetQScript { qscript: QScript =>
+trait AnnotationGff extends BiopetQScript with Reference { qscript: QScript =>
   /** GFF reference file in GFF3 format */
   lazy val annotationGff: File = {
-    val file: File = config("annotation_gff", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_gff_md5", freeVar = true))
+    val file: File = config("annotation_gff", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_gff_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   }
 }
 
-trait AnnotationRefFlat extends BiopetQScript { qscript: QScript =>
+trait AnnotationRefFlat extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val annotationRefFlat = new LazyCheck({
-    val file: File = config("annotation_refflat", freeVar = true)
-    inputFiles :+ InputFile(file, config("annotation_refflat_md5", freeVar = true))
+    val file: File = config("annotation_refflat", freeVar = true, extraSubPath = geneAnnotationSubPath)
+    inputFiles :+ InputFile(file, config("annotation_refflat_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
     file
   })
 }
 
-trait RibosomalRefFlat extends BiopetQScript { qscript: QScript =>
+trait RibosomalRefFlat extends BiopetQScript with Reference { qscript: QScript =>
   /** GTF reference file */
   lazy val ribosomalRefFlat = new LazyCheck({
-    val file: Option[File] = config("ribosome_refflat", freeVar = true)
+    val file: Option[File] = config("ribosome_refflat", freeVar = true, extraSubPath = geneAnnotationSubPath)
     file match {
-      case Some(f) => inputFiles :+ InputFile(f, config("ribosome_refflat_md5", freeVar = true))
+      case Some(f) => inputFiles :+ InputFile(f, config("ribosome_refflat_md5", freeVar = true, extraSubPath = geneAnnotationSubPath))
       case _       =>
     }
     file

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/ReferenceTest.scala

@@ -14,20 +14,20 @@
  */
 package nl.lumc.sasc.biopet.core
 
+import java.io.File
 import java.nio.file.Paths
 
 import nl.lumc.sasc.biopet.utils.{ ConfigUtils, Logging }
-import nl.lumc.sasc.biopet.utils.config.{ Configurable, Config }
+import nl.lumc.sasc.biopet.utils.config.{ Config, Configurable }
 import org.scalatest.Matchers
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.testng.TestNGSuite
-import org.mockito.Mockito._
 import org.testng.annotations.Test
 
 /**
  * Created by pjvan_thof on 12/30/15.
  */
-class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar {
+class ReferenceTest extends TestNGSuite with Matchers {
 
   import ReferenceTest._
 
@@ -58,6 +58,20 @@ class ReferenceTest extends TestNGSuite with Matchers with MockitoSugar {
     Logging.checkErrors(true)
   }
 
+  @Test
+  def testDbpsnp: Unit = {
+    val a = make(config :: Map("dbsnp_version" -> 1) :: testReferenceNoIndex :: Nil, fai = true, dict = true)
+    a.dbsnpVcfFile shouldBe Some(new File("vcf1"))
+    val b = make(config :: Map("dbsnp_version" -> 321) :: testReferenceNoIndex :: Nil, fai = true, dict = true)
+    b.dbsnpVcfFile shouldBe None
+
+    val c = make(config :: Map("dbsnp_version" -> 1) :: testReference :: Nil, fai = true, dict = true)
+    c.dbsnpVcfFile shouldBe None
+    val d = make(config :: Map("dbsnp_version" -> 2) :: testReference :: Nil, fai = true, dict = true)
+    d.dbsnpVcfFile shouldBe Some(new File("vcf2"))
+    val e = make(config :: Map("dbsnp_version" -> 3) :: testReference :: Nil, fai = true, dict = true)
+    e.dbsnpVcfFile shouldBe Some(new File("vcf3"))
+  }
 }
 
 object ReferenceTest {
@@ -72,13 +86,20 @@ object ReferenceTest {
     "references" -> Map(
       "test_species" -> Map(
         "test_genome" -> Map(
-          "reference_fasta" -> resourcePath("/fake_chrQ_no_index.fa")))))
+          "reference_fasta" -> resourcePath("/fake_chrQ_no_index.fa"),
+          "dbsnp_annotations" -> Map(
+            "1" -> Map("dbsnp_vcf" -> "vcf1")
+          )))))
 
   val testReference = Map(
     "references" -> Map(
       "test_species" -> Map(
         "test_genome" -> Map(
-          "reference_fasta" -> resourcePath("/fake_chrQ.fa")))))
+          "reference_fasta" -> resourcePath("/fake_chrQ.fa"),
+          "dbsnp_annotations" -> Map(
+            "2" -> Map("dbsnp_vcf" -> "vcf2"),
+            "3" -> Map("dbsnp_vcf" -> "vcf3")
+          )))))
 
   def make(configs: List[Map[String, Any]],
            r: Configurable = null,

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/annotation/AnnotationTest.scala

+package nl.lumc.sasc.biopet.core.annotation
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.BiopetQScript
+import nl.lumc.sasc.biopet.core.annotations.{ AnnotationGff, AnnotationGtf, AnnotationRefFlat }
+import nl.lumc.sasc.biopet.utils.Logging
+import nl.lumc.sasc.biopet.utils.config.Config
+import org.broadinstitute.gatk.queue.QScript
+import org.scalatest.Matchers
+import org.scalatest.testng.TestNGSuite
+import org.testng.annotations.Test
+
+/**
+ * Created by pjvanthof on 18/11/2016.
+ */
+class AnnotationTest extends TestNGSuite with Matchers {
+  @Test
+  def testAnnotationGtf: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationGtf
+    s1.annotationGtf shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationGtf
+    s2.annotationGtf shouldBe new File("no_set.gtf")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationGtf
+    s3.annotationGtf shouldBe new File("set1.gtf")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+
+  @Test
+  def testAnnotationGff: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationGff
+    s1.annotationGff shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationGff
+    s2.annotationGff shouldBe new File("no_set.gff")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationGff
+    s3.annotationGff shouldBe new File("set1.gff")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+
+  @Test
+  def testAnnotationRefFlat: Unit = {
+    val s1 = new AnnotationTest.Script(AnnotationTest.config) with AnnotationRefFlat
+    s1.annotationRefFlat.get shouldBe new File("")
+    an[IllegalStateException] shouldBe thrownBy(Logging.checkErrors())
+
+    val s2 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1")) with AnnotationRefFlat
+    s2.annotationRefFlat.get shouldBe new File("no_set.refFlat")
+    noException should be thrownBy (Logging.checkErrors())
+
+    val s3 = new AnnotationTest.Script(AnnotationTest.config ++ Map("species" -> "s1", "reference_name" -> "g1", "gene_annotation_name" -> "set1")) with AnnotationRefFlat
+    s3.annotationRefFlat.get shouldBe new File("set1.refFlat")
+    noException should be thrownBy (Logging.checkErrors())
+  }
+}
+
+object AnnotationTest {
+  class Script(c: Map[String, Any]) extends QScript with BiopetQScript {
+    override def globalConfig: Config = new Config(c)
+    val root = null
+
+    /** Init for pipeline */
+    def init(): Unit = ???
+
+    /** Pipeline itself */
+    def biopetScript(): Unit = ???
+  }
+
+  val config: Map[String, Any] = Map("references" -> Map("s1" -> Map("g1" -> Map(
+    "annotation_gtf" -> "no_set.gtf",
+    "annotation_gff" -> "no_set.gff",
+    "annotation_refflat" -> "no_set.refFlat",
+    "gene_annotations" -> Map("set1" -> Map(
+      "annotation_gtf" -> "set1.gtf",
+      "annotation_gff" -> "set1.gff",
+      "annotation_refflat" -> "set1.refFlat"
+    ))
+  ))))
+}
\ No newline at end of file

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/GffRead.scala

+package nl.lumc.sasc.biopet.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Version }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by pjvanthof on 20/06/16.
+ */
+class GffRead(val root: Configurable) extends BiopetCommandLineFunction {
+
+  executable = config("exe", default = "gffread", freeVar = false)
+
+  @Input
+  var input: File = _
+
+  @Output
+  var output: File = _
+
+  var T: Boolean = config("T", default = false, freeVar = false)
+
+  def cmdLine = executable +
+    (if (inputAsStdin) "" else required(input)) +
+    (if (outputAsStsout) "" else required("-o", output)) +
+    conditional(T, "-T")
+}

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Sed.scala

+package nl.lumc.sasc.biopet.extensions
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Version }
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+import scala.util.matching.Regex
+
+/**
+ * Created by pjvanthof on 18/05/16.
+ */
+class Sed(val root: Configurable) extends BiopetCommandLineFunction with Version {
+  executable = config("exe", default = "sed", freeVar = false)
+
+  /** Command to get version of executable */
+  override def versionCommand: String = executable + " --version"
+
+  /** Regex to get version from version command output */
+  override def versionRegex: Regex = """sed (GNU sed) \d+.\d+.\d+""".r
+
+  @Input(required = false)
+  var inputFile: File = _
+
+  @Output
+  var outputFile: File = _
+
+  var expressions: List[String] = Nil
+
+  def cmdLine = executable +
+    repeat("-e", expressions) +
+    (if (inputAsStdin) "" else required(inputFile)) +
+    (if (outputAsStsout) "" else " > " + required(outputFile))
+
+}

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/VariantEffectPredictor.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.extensions
 import java.io.File
 
 import nl.lumc.sasc.biopet.core.summary.Summarizable
-import nl.lumc.sasc.biopet.utils.{ Logging, VcfUtils, tryToParseNumber }
+import nl.lumc.sasc.biopet.utils.{ LazyCheck, Logging, VcfUtils, tryToParseNumber }
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, Reference, Version }
 import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
@@ -39,6 +39,16 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
   @Output(doc = "output file", required = true)
   var output: File = null
 
+  lazy val vepConfig = new LazyCheck({
+    val s: Option[String] = config("vep_config")
+    s
+  })
+
+  override def subPath = {
+    if (vepConfig.isSet) super.subPath ++ vepConfig()
+    else super.subPath
+  }
+
   def versionRegex = """version (\d*)""".r
   def versionCommand = executable + " " + vepScript + " --help"
 
@@ -106,7 +116,7 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
   var skibDbCheck: Boolean = config("skip_db_check", default = false)
 
   // Textual args
-  var vepConfig: Option[String] = config("config", freeVar = false)
+  var vepConfigArg: Option[String] = config("config", freeVar = false)
   var species: Option[String] = config("species", freeVar = false)
   var assembly: Option[String] = config("assembly")
   var format: Option[String] = config("format")
@@ -229,7 +239,7 @@ class VariantEffectPredictor(val root: Configurable) extends BiopetCommandLineFu
       conditional(lrg, "--lrg") +
       conditional(noWholeGenome, "--no_whole_genome") +
       conditional(skibDbCheck, "--skip_db_check") +
-      optional("--config", vepConfig) +
+      optional("--config", vepConfigArg) +
       optional("--species", species) +
       optional("--assembly", assembly) +
       optional("--format", format) +

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/BaseRecalibrator.scala

@@ -149,7 +149,7 @@ class BaseRecalibrator(val root: Configurable) extends CommandLineGATK with Scat
   @Argument(fullName = "filter_bases_not_stored", shortName = "filterNoBases", doc = "Filter out reads with no stored bases (i.e. '*' where the sequence should be), instead of failing with an error", required = false, exclusiveOf = "", validation = "")
   var filter_bases_not_stored: Boolean = config("filter_bases_not_stored", default = false)
 
-  if (config.contains("dbsnp")) knownSites :+= new File(config("dbsnp").asString)
+  dbsnpVcfFile.foreach(knownSites :+= _)
 
   override def beforeGraph() {
     super.beforeGraph()

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/CombineGVCFs.scala

@@ -37,7 +37,7 @@ class CombineGVCFs(val root: Configurable) extends CommandLineGATK with ScatterG
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** One or more input gVCF files */
   @Input(fullName = "variant", shortName = "V", doc = "One or more input gVCF files", required = true, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/GenotypeGVCFs.scala

@@ -102,7 +102,7 @@ class GenotypeGVCFs(val root: Configurable) extends CommandLineGATK with Scatter
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Filter out reads with CIGAR containing the N operator, instead of failing with an error */
   @Argument(fullName = "filter_reads_with_N_cigar", shortName = "filterRNC", doc = "Filter out reads with CIGAR containing the N operator, instead of failing with an error", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/HaplotypeCaller.scala

@@ -42,7 +42,7 @@ class HaplotypeCaller(val root: Configurable) extends CommandLineGATK with Scatt
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** If specified, we will not trim down the active region from the full region (active + extension) to just the active interval for genotyping */
   @Argument(fullName = "dontTrimActiveRegions", shortName = "dontTrimActiveRegions", doc = "If specified, we will not trim down the active region from the full region (active + extension) to just the active interval for genotyping", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/RealignerTargetCreator.scala

@@ -68,7 +68,7 @@ class RealignerTargetCreator(val root: Configurable) extends CommandLineGATK wit
   @Argument(fullName = "filter_bases_not_stored", shortName = "filterNoBases", doc = "Filter out reads with no stored bases (i.e. '*' where the sequence should be), instead of failing with an error", required = false, exclusiveOf = "", validation = "")
   var filter_bases_not_stored: Boolean = config("", default = false)
 
-  if (config.contains("dbsnp")) known :+= new File(config("dbsnp").asString)
+  dbsnpVcfFile.foreach(known :+= _)
 
   override def beforeGraph() {
     super.beforeGraph()

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/UnifiedGenotyper.scala

@@ -213,7 +213,7 @@ class UnifiedGenotyper(val root: Configurable) extends CommandLineGATK with Scat
 
   /** dbSNP file */
   @Input(fullName = "dbsnp", shortName = "D", doc = "dbSNP file", required = false, exclusiveOf = "", validation = "")
-  var dbsnp: Option[File] = config("dbsnp")
+  var dbsnp: Option[File] = dbsnpVcfFile
 
   /** Comparison VCF file */
   @Input(fullName = "comp", shortName = "comp", doc = "Comparison VCF file", required = false, exclusiveOf = "", validation = "")

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/VariantAnnotator.scala

@@ -37,7 +37,7 @@ class VariantAnnotator(val root: Configurable) extends CommandLineGATK with Scat