Merge pull request #165 from biopet/fix-BIOPET-750

Adding spark support for VcfStats

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/TemplateTool.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.core
 import java.io.{File, PrintWriter}
 
 import nl.lumc.sasc.biopet.utils.summary.Summary
-import nl.lumc.sasc.biopet.utils.{ConfigUtils, Question, ToolCommand}
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, Question, ToolCommand}
 
 import scala.io.Source
 
@@ -30,9 +30,8 @@ trait TemplateTool extends ToolCommand {
                   runScript: Option[File] = None,
                   expert: Boolean = false,
                   template: Option[File] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('o', "outputConfig") required () valueName "<file>" action { (x, c) =>
       c.copy(outputConfig = x)
     } text "Path to output config"

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/pipelinestatus/PipelineStatus.scala

@@ -20,7 +20,7 @@ import akka.actor.ActorSystem
 import akka.stream.ActorMaterializer
 import nl.lumc.sasc.biopet.utils.pim._
 import nl.lumc.sasc.biopet.utils.pim.{Job => PimJob}
-import nl.lumc.sasc.biopet.utils.{ConfigUtils, ToolCommand}
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, ToolCommand}
 import play.api.libs.ws.ahc.AhcWSClient
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -41,9 +41,8 @@ object PipelineStatus extends ToolCommand {
                   compressPlots: Boolean = true,
                   pimHost: Option[String] = None,
                   pimRunId: Option[String] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('d', "pipelineDir") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
       c.copy(pipelineDir = x)
     } text "Output directory of the pipeline"

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -17,10 +17,10 @@ package nl.lumc.sasc.biopet.core.report
 import java.io._
 
 import nl.lumc.sasc.biopet.core.ToolCommandFunction
-import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Sample, Run}
+import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Run, Sample}
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
 import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{LibraryId, SampleId}
-import nl.lumc.sasc.biopet.utils.{IoUtils, Logging, ToolCommand}
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, IoUtils, Logging, ToolCommand}
 import org.broadinstitute.gatk.utils.commandline.Input
 import org.fusesource.scalate.TemplateEngine
 
@@ -82,9 +82,8 @@ trait ReportBuilder extends ToolCommand {
                   outputDir: File = null,
                   runId: Int = 0,
                   pageArgs: mutable.Map[String, Any] = mutable.Map())
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
 
     head(
       s"""

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/clever/CleverFixVCF.scala

@@ -20,7 +20,7 @@ package nl.lumc.sasc.biopet.extensions.clever
 import java.io.{File, PrintWriter}
 
 import nl.lumc.sasc.biopet.core.BiopetJavaCommandLineFunction
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.utils.commandline.{Argument, Input, Output}
 
@@ -48,9 +48,8 @@ class CleverFixVCF(val parent: Configurable) extends BiopetJavaCommandLineFuncti
 
 object CleverFixVCF extends ToolCommand {
   case class Args(inputVCF: File = null, sampleLabel: String = "", outputVCF: File = null)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('i', "inputvcf") required () valueName "<vcffile/path>" action { (x, c) =>
       c.copy(inputVCF = x)
     } text "Please specify the input Clever VCF file"

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/pindel/PindelConfig.scala

@@ -18,7 +18,7 @@ import java.io.{File, PrintWriter}
 import htsjdk.samtools.SamReaderFactory
 import nl.lumc.sasc.biopet.core.BiopetJavaCommandLineFunction
 import nl.lumc.sasc.biopet.utils.config.Configurable
-import nl.lumc.sasc.biopet.utils.{BamUtils, ToolCommand}
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, BamUtils, ToolCommand}
 import org.broadinstitute.gatk.utils.commandline.{Argument, Input, Output}
 
 class PindelConfig(val parent: Configurable) extends BiopetJavaCommandLineFunction {
@@ -46,9 +46,8 @@ object PindelConfig extends ToolCommand {
                   sampleLabel: Option[String] = None,
                   insertSize: Option[Int] = None,
                   output: Option[File] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('i', "inputbam") required () valueName "<bamfile/path>" action { (x, c) =>
       c.copy(inputBam = x)
     } text "Please specify the input bam file"

biopet-package/pom.xml

@@ -155,6 +155,20 @@
                         </transformer>
                     </transformers>
                     <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                        <filter>
+                            <artifact>javax.servlet:servlet-api:2.5</artifact>
+                            <excludes>
+                                <exclude>*.*</exclude>
+                            </excludes>
+                        </filter>
                     </filters>
                 </configuration>
                 <executions>

biopet-tools/pom.xml

@@ -29,6 +29,11 @@
     <artifactId>BiopetTools</artifactId>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_2.11</artifactId>
+            <version>2.2.0</version>
+        </dependency>
         <dependency>
             <groupId>nl.lumc.sasc</groupId>
             <artifactId>BiopetUtils</artifactId>

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/AnnotateVcfWithBed.scala

@@ -22,7 +22,7 @@ import htsjdk.variant.variantcontext.writer.{
   VariantContextWriterBuilder
 }
 import htsjdk.variant.vcf.{VCFFileReader, VCFHeaderLineCount, VCFHeaderLineType, VCFInfoHeaderLine}
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 import nl.lumc.sasc.biopet.utils.intervals.{BedRecord, BedRecordList}
 
 import scala.collection.JavaConversions._
@@ -51,9 +51,8 @@ object AnnotateVcfWithBed extends ToolCommand {
                   fieldName: String = null,
                   fieldDescription: String = "",
                   fieldType: String = "String")
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "inputFile") required () unbounded () valueName "<vcf file>" action { (x, c) =>
       c.copy(inputFile = x)
     } text "Input VCF file. Mandatory field"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/BaseCounter.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
 import java.io.{File, PrintWriter}
 
 import htsjdk.samtools.{SAMRecord, SamReaderFactory}
-import nl.lumc.sasc.biopet.utils.{ConfigUtils, ToolCommand}
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, ToolCommand}
 import nl.lumc.sasc.biopet.utils.intervals.{BedRecord, BedRecordList}
 import picard.annotation.{Gene, GeneAnnotationReader}
 
@@ -35,9 +35,8 @@ object BaseCounter extends ToolCommand {
                   outputDir: File = null,
                   bamFile: File = null,
                   prefix: String = "output")
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('r', "refFlat") required () valueName "<file>" action { (x, c) =>
       c.copy(refFlat = x)
     } text "refFlat file. Mandatory"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/BastyGenerateFasta.scala

@@ -20,7 +20,7 @@ import htsjdk.samtools.SamReaderFactory
 import htsjdk.samtools.reference.IndexedFastaSequenceFile
 import htsjdk.variant.variantcontext.VariantContext
 import htsjdk.variant.vcf.VCFFileReader
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 import nl.lumc.sasc.biopet.utils.VcfUtils._
 
 import scala.collection.JavaConversions._
@@ -38,9 +38,8 @@ object BastyGenerateFasta extends ToolCommand {
                   minAD: Int = 8,
                   minDepth: Int = 8,
                   reference: File = null)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('V', "inputVcf") unbounded () valueName "<file>" action { (x, c) =>
       c.copy(inputVcf = x)
     } text "vcf file, needed for outputVariants and outputConsensusVariants" validate { x =>

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/BedToInterval.scala

@@ -16,10 +16,10 @@ package nl.lumc.sasc.biopet.tools
 
 import java.io.{File, PrintWriter}
 
-import htsjdk.samtools.{SAMSequenceRecord, SamReaderFactory}
-import nl.lumc.sasc.biopet.utils.ToolCommand
-import scala.collection.JavaConversions._
+import htsjdk.samtools.SamReaderFactory
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
+import scala.collection.JavaConversions._
 import scala.io.Source
 
 /**
@@ -28,9 +28,8 @@ import scala.io.Source
 object BedToInterval extends ToolCommand {
 
   case class Args(inputFile: File = null, outputFile: File = null, bamFile: File = null)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
       c.copy(inputFile = x)
     }

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/BedtoolsCoverageToCounts.scala

@@ -16,15 +16,15 @@ package nl.lumc.sasc.biopet.tools
 
 import java.io.{File, PrintWriter}
 
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.collection.{SortedMap, mutable}
 import scala.io.Source
 
 object BedtoolsCoverageToCounts extends ToolCommand {
-  case class Args(input: File = null, output: File = null) extends AbstractArgs
+  case class Args(input: File = null, output: File = null)
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "input") required () valueName "<file>" action { (x, c) =>
       c.copy(input = x)
     } text "Coverage file produced with bedtools"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/CheckAllelesVcfInBam.scala

@@ -23,7 +23,7 @@ import htsjdk.variant.variantcontext.writer.{
   VariantContextWriterBuilder
 }
 import htsjdk.variant.vcf.{VCFFileReader, VCFHeaderLineCount, VCFHeaderLineType, VCFInfoHeaderLine}
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable
@@ -49,9 +49,8 @@ object CheckAllelesVcfInBam extends ToolCommand {
                   samples: List[String] = Nil,
                   bamFiles: List[File] = Nil,
                   minMapQual: Int = 1)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "inputFile") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
       c.copy(inputFile = x)
     } text "VCF file"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/DownloadNcbiAssembly.scala

@@ -16,7 +16,7 @@ package nl.lumc.sasc.biopet.tools
 
 import java.io.{File, PrintWriter}
 
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.io.Source
 
@@ -31,9 +31,8 @@ object DownloadNcbiAssembly extends ToolCommand {
                   contigNameHeader: Option[String] = None,
                   mustHaveOne: List[(String, String)] = List(),
                   mustNotHave: List[(String, String)] = List())
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('a', "assembly_report") required () unbounded () valueName "<file>" action {
       (x, c) =>
         c.copy(assemblyReport = x)

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/ExtractAlignedFastq.scala

@@ -19,7 +19,7 @@ import java.io.File
 import htsjdk.samtools.{QueryInterval, SamReaderFactory, ValidationStringency}
 import htsjdk.samtools.fastq.{BasicFastqWriter, FastqReader, FastqRecord}
 import htsjdk.samtools.util.Interval
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{Set => MSet}
@@ -181,10 +181,9 @@ object ExtractAlignedFastq extends ToolCommand {
                   outputFastq2: Option[File] = None,
                   minMapQ: Int = 0,
                   commonSuffixLength: Int = 0)
-      extends AbstractArgs
 
   /** Command line argument parser */
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
 
     head(s"""
         |$commandName - Select aligned FASTQ records

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/ExtractTagsFromGtf.scala

@@ -2,7 +2,7 @@ package nl.lumc.sasc.biopet.tools
 
 import java.io.{File, PrintWriter}
 
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 import nl.lumc.sasc.biopet.utils.annotation.Feature
 
 import scala.io.Source
@@ -15,9 +15,8 @@ object ExtractTagsFromGtf extends ToolCommand {
                   gtfFile: File = null,
                   tags: List[String] = Nil,
                   feature: Option[String] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('o', "output") required () unbounded () valueName "<file>" action { (x, c) =>
       c.copy(outputFile = x)
     } text "Input refFlat file. Mandatory"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FastqFilter.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
 import java.io.File
 
 import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader}
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.util.matching.Regex
 import scala.collection.JavaConversions._
@@ -33,9 +33,8 @@ object FastqFilter extends ToolCommand {
     * @param outputFile output fastq files
     */
   case class Args(inputFile: File = null, outputFile: File = null, idRegex: Option[Regex] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
       c.copy(inputFile = x)
     } text "Path to input file"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FastqSplitter.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
 import java.io.File
 
 import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader}
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 object FastqSplitter extends ToolCommand {
 
@@ -26,9 +26,9 @@ object FastqSplitter extends ToolCommand {
     * @param inputFile input fastq file
     * @param outputFile output fastq files
     */
-  case class Args(inputFile: File = null, outputFile: List[File] = Nil) extends AbstractArgs
+  case class Args(inputFile: File = null, outputFile: List[File] = Nil)
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
       c.copy(inputFile = x)
     } text "Path to input file"

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FastqSync.scala

@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
 import java.io.File
 
 import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader, FastqRecord}
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.annotation.tailrec
 import scala.collection.JavaConverters._
@@ -113,9 +113,8 @@ object FastqSync extends ToolCommand {
                   inputFastq2: File = null,
                   outputFastq1: File = null,
                   outputFastq2: File = null)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
 
     head(s"""
         |$commandName - Sync paired-end FASTQ files.

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FindOverlapMatch.scala

@@ -16,7 +16,7 @@ package nl.lumc.sasc.biopet.tools
 
 import java.io.{File, PrintStream}
 
-import nl.lumc.sasc.biopet.utils.ToolCommand
+import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
 
 import scala.collection.mutable.ListBuffer
 import scala.io.Source
@@ -35,9 +35,8 @@ object FindOverlapMatch extends ToolCommand {
                   filterSameNames: Boolean = true,
                   rowSampleRegex: Option[Regex] = None,
                   columnSampleRegex: Option[Regex] = None)
-      extends AbstractArgs
 
-  class OptParser extends AbstractOptParser {
+  class OptParser extends AbstractOptParser[Args](commandName) {
     opt[File]('i', "input") required () unbounded () valueName "<file>" action { (x, c) =>
       c.copy(inputMetrics = x)
     } text "Input should be a table where the first row and column have the ID's, those can be different"