Merge branch 'feature-varda' into 'develop'

Feature varda

fixes #208 

This is to track this branch in jenkins

See merge request !245

public/biopet-extensions/pom.xml

@@ -46,6 +46,13 @@
             <version>2.2.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <version>1.9.5</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
 </project>
\ No newline at end of file

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/bcftools/BcftoolsView.scala

+package nl.lumc.sasc.biopet.extensions.bcftools
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Input, Output }
+
+/**
+ * Created by ahbbollen on 12-10-15.
+ */
+class BcftoolsView(val root: Configurable) extends Bcftools {
+
+  @Input(doc = "Input VCF file")
+  var input: File = _
+
+  @Output(doc = "Output file")
+  var output: File = _
+
+  @Argument(doc = "drop individual genotypes", required = false)
+  var dropGenotype: Boolean = config("drop_genotype", default = false)
+
+  @Argument(doc = "header only", required = false)
+  var headerOnly: Boolean = config("header_only", false)
+
+  @Argument(doc = "Compression level", required = false)
+  var compressionLevel: Int = config("compression_level", default = 9)
+
+  @Argument(doc = "output type", required = false)
+  var outputType: String = "z"
+
+  @Argument(doc = "regions", required = false)
+  var regions: Option[String] = config("r")
+
+  @Argument(doc = "region file", required = false)
+  var regionFile: Option[File] = config("R")
+
+  @Argument(doc = "targets", required = false)
+  var targets: Option[String] = config("t")
+
+  @Argument(doc = "targets file", required = false)
+  var targetFile: Option[File] = config("T")
+
+  @Argument(doc = "trim alt alleles", required = false)
+  var trimAltAlleles: Boolean = config("trim_alt_allele", default = false)
+
+  @Argument(doc = "no update", required = false)
+  var noUpdate: Boolean = config("no_update", default = false)
+
+  @Argument(doc = "samples", required = false)
+  var samples: List[String] = config("s", default = Nil)
+
+  @Argument(doc = "samples file", required = false)
+  var sampleFile: Option[File] = config("S")
+
+  @Argument(doc = "minimum allele count", required = false)
+  var minAC: Option[Int] = config("c")
+
+  @Argument(doc = "max allele count", required = false)
+  var maxAC: Option[Int] = config("C")
+
+  @Argument(doc = "exclude (expression)", required = false)
+  var exclude: Option[String] = config("e")
+
+  @Argument(doc = "apply filters", required = false)
+  var applyFilters: List[String] = config("F", default = Nil)
+
+  @Argument(doc = "genotype", required = false)
+  var genotype: Option[String] = config("g")
+
+  @Argument(doc = "include (expression)", required = false)
+  var include: Option[String] = config("i")
+
+  @Argument(doc = "Known (ID field is not .) only", required = false)
+  var known: Boolean = config("k", default = false)
+
+  @Argument(doc = "min alleles", required = false)
+  var minAlleles: Option[Int] = config("m")
+
+  @Argument(doc = "max alleles", required = false)
+  var maxAlleles: Option[Int] = config("M")
+
+  @Argument(doc = "novel (ID field is .) only", required = false)
+  var novel: Boolean = config("n", false)
+
+  @Argument(doc = "phased only", required = false)
+  var phased: Boolean = config("p", false)
+
+  @Argument(doc = "exclude phased (only)", required = false)
+  var excludePhased: Boolean = config("P", false)
+
+  @Argument(doc = "min allele frequency", required = false)
+  var minAF: Option[Int] = config("q")
+
+  @Argument(doc = "max allele frequency", required = false)
+  var maxAF: Option[Int] = config("Q")
+
+  @Argument(doc = "uncalled only", required = false)
+  var uncalled: Boolean = config("u", default = false)
+
+  @Argument(doc = "exclude uncalled (only)", required = false)
+  var excludeUncalled: Boolean = config("U", default = false)
+
+  @Argument(doc = "types", required = false)
+  var types: Option[String] = config("v")
+
+  @Argument(doc = "exclude types", required = false)
+  var excludeTypes: Option[String] = config("V")
+
+  @Argument(doc = "private (requires samples)", required = false)
+  var onlyPrivate: Boolean = config("x", default = false)
+
+  @Argument(doc = "Exclude privates", required = false)
+  var excludePrivate: Boolean = config("X", default = false)
+
+  override def beforeGraph() = {
+    super.beforeGraph()
+
+    require((compressionLevel <= 9) && (compressionLevel >= 0))
+    require(
+      (outputType.length == 1) &&
+        (outputType == "z" || outputType == "b" || outputType == "u" || outputType == "v")
+    )
+  }
+
+  def baseCmd = {
+    executable +
+      required("view") +
+      conditional(dropGenotype, "-G") +
+      conditional(headerOnly, "-h") +
+      required("-l", compressionLevel) +
+      required("-O", outputType) +
+      optional("-r", regions) +
+      optional("-R", regionFile) +
+      optional("-t", targets) +
+      optional("-T", targetFile) +
+      conditional(trimAltAlleles, "-a") +
+      conditional(noUpdate, "-I") +
+      repeat("-s", samples) +
+      optional("-S", sampleFile) +
+      optional("-c", minAC) +
+      optional("-C", maxAC) +
+      optional("-e", exclude) +
+      optional("-f", applyFilters) +
+      optional("-g", genotype) +
+      optional("-i", include) +
+      conditional(known, "-k") +
+      optional("-m", minAlleles) +
+      optional("-M", maxAlleles) +
+      conditional(novel, "-n") +
+      conditional(phased, "-p") +
+      conditional(excludePhased, "-P") +
+      optional("-q", minAF) +
+      optional("-Q", maxAF) +
+      conditional(uncalled, "-u") +
+      conditional(excludeUncalled, "-U") +
+      optional("-v", types) +
+      optional("-V", excludeTypes) +
+      conditional(onlyPrivate, "-x") +
+      conditional(excludePrivate, "-X")
+  }
+
+  def cmdPipeInput = {
+    baseCmd + "-"
+  }
+
+  def cmdPipe = {
+    baseCmd + required(input)
+  }
+
+  def cmdLine = {
+    baseCmd + required("-o", output) + required(input)
+  }
+
+  /**
+   * Convert cmdLine into line without quotes and double spaces
+   * primarily for testing
+   * @return
+   */
+  final def cmd = {
+    val a = cmdLine
+    a.replace("'", "").replace("  ", " ").trim
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/bedtools/BedtoolsMerge.scala

+package nl.lumc.sasc.biopet.extensions.bedtools
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output, Input }
+
+/**
+ * Created by ahbbollen on 5-1-16.
+ */
+class BedtoolsMerge(val root: Configurable) extends Bedtools {
+
+  @Input(doc = "Input bed file")
+  var input: File = _
+
+  @Argument(doc = "Distance")
+  var dist: Option[Int] = config("dist") //default of tool is 1
+
+  @Output(doc = "Output bed file")
+  var output: File = _
+
+  def cmdLine = {
+    required(executable) + required("merge") +
+      required("-i", input) + optional("-d", dist) +
+      " > " + required(output)
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/Manwe.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.{ PrintWriter, File }
+
+import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
+import org.broadinstitute.gatk.utils.commandline.{ Output, Argument }
+
+/**
+ * Created by ahbbollen on 23-9-15.
+ * This is python, but not accessed like a script; i.e. called by simply
+ * manwe [subcommand]
+ */
+abstract class Manwe extends BiopetCommandLineFunction {
+  executable = config("exe", default = "manwe", submodule = "manwe")
+
+  var manweConfig: File = createManweConfig(None)
+
+  @Output(doc = "the output file")
+  var output: File = _
+
+  var manweHelp: Boolean = false
+
+  def subCommand: String
+
+  final def cmdLine = {
+    manweConfig = createManweConfig(Option(output).map(_.getParentFile))
+    required(executable) +
+      subCommand +
+      required("-c", manweConfig) +
+      conditional(manweHelp, "-h") +
+      " > " +
+      required(output)
+
+  }
+
+  /**
+   * Convert cmdLine into line without quotes and double spaces
+   * primarily for testing
+   * @return
+   */
+  final def cmd = {
+    val a = cmdLine
+    a.replace("'", "").replace("  ", " ").trim
+  }
+
+  /**
+   * Create Manwe config from biopet config
+   * @return Manwe config file
+   */
+  def createManweConfig(directory: Option[File]): File = {
+    val url: String = config("varda_root")
+    val token: String = config("varda_token")
+    val sslSettings: Option[String] = config("varda_verify_certificate")
+    val collectionCacheSize: Option[Int] = config("varda_cache_size", default = 20)
+    val dataBufferSize: Option[Int] = config("varda_buffer_size", default = 1024 * 1024)
+    val taskPollWait: Option[Int] = config("varda_task_poll_wait", default = 2)
+
+    val settingsMap: Map[String, Any] = Map(
+      "API_ROOT" -> s"'$url'",
+      "TOKEN" -> s"'$token'",
+      "VERIFY_CERTIFICATE" -> (sslSettings match {
+        case Some("true")  => "True"
+        case Some("false") => "False"
+        case Some(x)       => s"'$x'"
+        case _             => "True"
+      }),
+      "COLLECTION_CACHE_SIZE" -> collectionCacheSize.getOrElse(20),
+      "DATA_BUFFER_SIZE" -> dataBufferSize.getOrElse(1048576),
+      "TASK_POLL_WAIT" -> taskPollWait.getOrElse(2)
+    )
+
+    val file = directory match {
+      case Some(dir) => File.createTempFile("manwe_config", ".py", dir)
+      case None      => File.createTempFile("manwe_config", ".py")
+    }
+
+    file.deleteOnExit()
+    val writer = new PrintWriter(file)
+    settingsMap.foreach { case (key, value) => writer.println(s"$key = $value") }
+    writer.close()
+    file
+  }
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweAnnotateBed.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Argument }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweAnnotateBed(val root: Configurable) extends Manwe {
+
+  @Input(doc = "the bed to annotate")
+  var bed: File = _
+
+  @Argument(doc = "flag if data has already been uploaded")
+  var alreadyUploaded: Boolean = false
+
+  @Argument(doc = "Flag whether to wait for annotation to complete on the server")
+  var waitToComplete: Boolean = false
+
+  @Argument(doc = "annotation queries", required = false)
+  var queries: List[String] = Nil
+
+  def subCommand = {
+    required("annotate-bed") + required(bed) +
+      conditional(alreadyUploaded, "-u") +
+      repeat("-q", queries) +
+      conditional(waitToComplete, "--wait")
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweAnnotateVcf.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Input }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweAnnotateVcf(val root: Configurable) extends Manwe {
+
+  @Input(doc = "the vcf to annotate")
+  var vcf: File = _
+
+  @Argument(doc = "flag if data has already been uploaded")
+  var alreadyUploaded: Boolean = false
+
+  @Argument(doc = "flag whether to wait for annotation to complete")
+  var waitToComplete: Boolean = false
+
+  @Argument(doc = "annotation queries", required = false)
+  var queries: List[String] = Nil
+
+  def subCommand = {
+    required("annotate-vcf") + required(vcf) +
+      conditional(alreadyUploaded, "-u") +
+      repeat("-q", queries) +
+      conditional(waitToComplete, "--wait")
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweDataSourcesAnnotate.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.Argument
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweDataSourcesAnnotate(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "uri to data source to annotate")
+  var uri: Option[String] = _
+
+  @Argument(doc = "list of queries", required = false)
+  var queries: List[String] = Nil
+
+  @Argument(doc = "Flag whether to wait for annotation to complete on server")
+  var waitToComplete: Boolean = false
+
+  def subCommand = {
+    required("data-sources") + required("annotate") +
+      required(uri) +
+      repeat("-q", queries) +
+      conditional(waitToComplete, "--wait")
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweDataSourcesDownload.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.Argument
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweDataSourcesDownload(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "uri to data source to download")
+  var uri: String = _
+
+  def subCommand = {
+    required("data-sources") +
+      required("download") +
+      required(uri)
+  }
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweDataSourcesList.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweDataSourcesList(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "User uri to filter by")
+  var user: Option[String] = _
+
+  def subCommand = {
+    required("data-sources") +
+      required("list") +
+      optional("-u", user)
+  }
+
+}
+

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweDataSourcesShow.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweDataSourcesShow(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "uri of data source")
+  var uri: Option[String] = _
+
+  def subCommand = {
+    required("data-sources") +
+      required("show") +
+      required(uri)
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweSamplesActivate.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweSamplesActivate(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "uri to sample to activate")
+  var uri: String = _
+
+  def subCommand = {
+    required("samples") +
+      required("activate") +
+      required(uri)
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweSamplesAdd.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweSamplesAdd(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "the sample name")
+  var name: Option[String] = _
+
+  @Argument(doc = "the sample groups [uris]", required = false)
+  var group: List[String] = Nil
+
+  @Argument(doc = "pool size")
+  var poolSize: Option[Int] = _
+
+  def subCommand = {
+    required("samples") +
+      required("add") +
+      required(name) +
+      optional("-s", poolSize) +
+      repeat("-g", group)
+  }
+
+}

public/biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/manwe/ManweSamplesAnnotateVariations.scala

+package nl.lumc.sasc.biopet.extensions.manwe
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Argument, Output }
+
+/**
+ * Created by ahbbollen on 24-9-15.
+ */
+class ManweSamplesAnnotateVariations(val root: Configurable) extends Manwe {
+
+  @Argument(doc = "uri to sample to annotate")
+  var uri: Option[String] = _
+
+  @Argument(doc = "Annotation queries", required = false)
+  var queries: List[String] = Nil
+
+  def subCommand = {
+    required("samples") +
+      required("annotate-variations") +
+      required(uri) +
+      repeat("-q", queries)
+  }