Commit c913b602 authored by Peter van 't Hof's avatar Peter van 't Hof Committed by GitHub

Merge pull request #165 from biopet/fix-BIOPET-750

Adding spark support for VcfStats
parents dc145917 db65e3d7
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.core
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.summary.Summary
import nl.lumc.sasc.biopet.utils.{ConfigUtils, Question, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, Question, ToolCommand}
import scala.io.Source
......@@ -30,9 +30,8 @@ trait TemplateTool extends ToolCommand {
runScript: Option[File] = None,
expert: Boolean = false,
template: Option[File] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('o', "outputConfig") required () valueName "<file>" action { (x, c) =>
c.copy(outputConfig = x)
} text "Path to output config"
......
......@@ -20,7 +20,7 @@ import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import nl.lumc.sasc.biopet.utils.pim._
import nl.lumc.sasc.biopet.utils.pim.{Job => PimJob}
import nl.lumc.sasc.biopet.utils.{ConfigUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, ToolCommand}
import play.api.libs.ws.ahc.AhcWSClient
import scala.concurrent.ExecutionContext.Implicits.global
......@@ -41,9 +41,8 @@ object PipelineStatus extends ToolCommand {
compressPlots: Boolean = true,
pimHost: Option[String] = None,
pimRunId: Option[String] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('d', "pipelineDir") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(pipelineDir = x)
} text "Output directory of the pipeline"
......
......@@ -17,10 +17,10 @@ package nl.lumc.sasc.biopet.core.report
import java.io._
import nl.lumc.sasc.biopet.core.ToolCommandFunction
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Sample, Run}
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Module, Pipeline, Run, Sample}
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{LibraryId, SampleId}
import nl.lumc.sasc.biopet.utils.{IoUtils, Logging, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, IoUtils, Logging, ToolCommand}
import org.broadinstitute.gatk.utils.commandline.Input
import org.fusesource.scalate.TemplateEngine
......@@ -82,9 +82,8 @@ trait ReportBuilder extends ToolCommand {
outputDir: File = null,
runId: Int = 0,
pageArgs: mutable.Map[String, Any] = mutable.Map())
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
head(
s"""
......
......@@ -20,7 +20,7 @@ package nl.lumc.sasc.biopet.extensions.clever
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.core.BiopetJavaCommandLineFunction
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import nl.lumc.sasc.biopet.utils.config.Configurable
import org.broadinstitute.gatk.utils.commandline.{Argument, Input, Output}
......@@ -48,9 +48,8 @@ class CleverFixVCF(val parent: Configurable) extends BiopetJavaCommandLineFuncti
object CleverFixVCF extends ToolCommand {
case class Args(inputVCF: File = null, sampleLabel: String = "", outputVCF: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('i', "inputvcf") required () valueName "<vcffile/path>" action { (x, c) =>
c.copy(inputVCF = x)
} text "Please specify the input Clever VCF file"
......
......@@ -18,7 +18,7 @@ import java.io.{File, PrintWriter}
import htsjdk.samtools.SamReaderFactory
import nl.lumc.sasc.biopet.core.BiopetJavaCommandLineFunction
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.{BamUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, BamUtils, ToolCommand}
import org.broadinstitute.gatk.utils.commandline.{Argument, Input, Output}
class PindelConfig(val parent: Configurable) extends BiopetJavaCommandLineFunction {
......@@ -46,9 +46,8 @@ object PindelConfig extends ToolCommand {
sampleLabel: Option[String] = None,
insertSize: Option[Int] = None,
output: Option[File] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('i', "inputbam") required () valueName "<bamfile/path>" action { (x, c) =>
c.copy(inputBam = x)
} text "Please specify the input bam file"
......
......@@ -155,6 +155,20 @@
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
<filter>
<artifact>javax.servlet:servlet-api:2.5</artifact>
<excludes>
<exclude>*.*</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
......
......@@ -29,6 +29,11 @@
<artifactId>BiopetTools</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>nl.lumc.sasc</groupId>
<artifactId>BiopetUtils</artifactId>
......
......@@ -22,7 +22,7 @@ import htsjdk.variant.variantcontext.writer.{
VariantContextWriterBuilder
}
import htsjdk.variant.vcf.{VCFFileReader, VCFHeaderLineCount, VCFHeaderLineType, VCFInfoHeaderLine}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import nl.lumc.sasc.biopet.utils.intervals.{BedRecord, BedRecordList}
import scala.collection.JavaConversions._
......@@ -51,9 +51,8 @@ object AnnotateVcfWithBed extends ToolCommand {
fieldName: String = null,
fieldDescription: String = "",
fieldType: String = "String")
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputFile") required () unbounded () valueName "<vcf file>" action { (x, c) =>
c.copy(inputFile = x)
} text "Input VCF file. Mandatory field"
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import htsjdk.samtools.{SAMRecord, SamReaderFactory}
import nl.lumc.sasc.biopet.utils.{ConfigUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ConfigUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.intervals.{BedRecord, BedRecordList}
import picard.annotation.{Gene, GeneAnnotationReader}
......@@ -35,9 +35,8 @@ object BaseCounter extends ToolCommand {
outputDir: File = null,
bamFile: File = null,
prefix: String = "output")
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('r', "refFlat") required () valueName "<file>" action { (x, c) =>
c.copy(refFlat = x)
} text "refFlat file. Mandatory"
......
......@@ -20,7 +20,7 @@ import htsjdk.samtools.SamReaderFactory
import htsjdk.samtools.reference.IndexedFastaSequenceFile
import htsjdk.variant.variantcontext.VariantContext
import htsjdk.variant.vcf.VCFFileReader
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import nl.lumc.sasc.biopet.utils.VcfUtils._
import scala.collection.JavaConversions._
......@@ -38,9 +38,8 @@ object BastyGenerateFasta extends ToolCommand {
minAD: Int = 8,
minDepth: Int = 8,
reference: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('V', "inputVcf") unbounded () valueName "<file>" action { (x, c) =>
c.copy(inputVcf = x)
} text "vcf file, needed for outputVariants and outputConsensusVariants" validate { x =>
......
......@@ -16,10 +16,10 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import htsjdk.samtools.{SAMSequenceRecord, SamReaderFactory}
import nl.lumc.sasc.biopet.utils.ToolCommand
import scala.collection.JavaConversions._
import htsjdk.samtools.SamReaderFactory
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.JavaConversions._
import scala.io.Source
/**
......@@ -28,9 +28,8 @@ import scala.io.Source
object BedToInterval extends ToolCommand {
case class Args(inputFile: File = null, outputFile: File = null, bamFile: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
c.copy(inputFile = x)
}
......
......@@ -16,15 +16,15 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.{SortedMap, mutable}
import scala.io.Source
object BedtoolsCoverageToCounts extends ToolCommand {
case class Args(input: File = null, output: File = null) extends AbstractArgs
case class Args(input: File = null, output: File = null)
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "input") required () valueName "<file>" action { (x, c) =>
c.copy(input = x)
} text "Coverage file produced with bedtools"
......
......@@ -23,7 +23,7 @@ import htsjdk.variant.variantcontext.writer.{
VariantContextWriterBuilder
}
import htsjdk.variant.vcf.{VCFFileReader, VCFHeaderLineCount, VCFHeaderLineType, VCFInfoHeaderLine}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.JavaConversions._
import scala.collection.mutable
......@@ -49,9 +49,8 @@ object CheckAllelesVcfInBam extends ToolCommand {
samples: List[String] = Nil,
bamFiles: List[File] = Nil,
minMapQual: Int = 1)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputFile") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(inputFile = x)
} text "VCF file"
......
......@@ -16,7 +16,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.io.Source
......@@ -31,9 +31,8 @@ object DownloadNcbiAssembly extends ToolCommand {
contigNameHeader: Option[String] = None,
mustHaveOne: List[(String, String)] = List(),
mustNotHave: List[(String, String)] = List())
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('a', "assembly_report") required () unbounded () valueName "<file>" action {
(x, c) =>
c.copy(assemblyReport = x)
......
......@@ -19,7 +19,7 @@ import java.io.File
import htsjdk.samtools.{QueryInterval, SamReaderFactory, ValidationStringency}
import htsjdk.samtools.fastq.{BasicFastqWriter, FastqReader, FastqRecord}
import htsjdk.samtools.util.Interval
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.JavaConverters._
import scala.collection.mutable.{Set => MSet}
......@@ -181,10 +181,9 @@ object ExtractAlignedFastq extends ToolCommand {
outputFastq2: Option[File] = None,
minMapQ: Int = 0,
commonSuffixLength: Int = 0)
extends AbstractArgs
/** Command line argument parser */
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
head(s"""
|$commandName - Select aligned FASTQ records
......
......@@ -2,7 +2,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import nl.lumc.sasc.biopet.utils.annotation.Feature
import scala.io.Source
......@@ -15,9 +15,8 @@ object ExtractTagsFromGtf extends ToolCommand {
gtfFile: File = null,
tags: List[String] = Nil,
feature: Option[String] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('o', "output") required () unbounded () valueName "<file>" action { (x, c) =>
c.copy(outputFile = x)
} text "Input refFlat file. Mandatory"
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.File
import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.util.matching.Regex
import scala.collection.JavaConversions._
......@@ -33,9 +33,8 @@ object FastqFilter extends ToolCommand {
* @param outputFile output fastq files
*/
case class Args(inputFile: File = null, outputFile: File = null, idRegex: Option[Regex] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
c.copy(inputFile = x)
} text "Path to input file"
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.File
import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
object FastqSplitter extends ToolCommand {
......@@ -26,9 +26,9 @@ object FastqSplitter extends ToolCommand {
* @param inputFile input fastq file
* @param outputFile output fastq files
*/
case class Args(inputFile: File = null, outputFile: List[File] = Nil) extends AbstractArgs
case class Args(inputFile: File = null, outputFile: List[File] = Nil)
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputFile") required () valueName "<file>" action { (x, c) =>
c.copy(inputFile = x)
} text "Path to input file"
......
......@@ -17,7 +17,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.File
import htsjdk.samtools.fastq.{AsyncFastqWriter, BasicFastqWriter, FastqReader, FastqRecord}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.annotation.tailrec
import scala.collection.JavaConverters._
......@@ -113,9 +113,8 @@ object FastqSync extends ToolCommand {
inputFastq2: File = null,
outputFastq1: File = null,
outputFastq2: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
head(s"""
|$commandName - Sync paired-end FASTQ files.
......
......@@ -16,7 +16,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintStream}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.mutable.ListBuffer
import scala.io.Source
......@@ -35,9 +35,8 @@ object FindOverlapMatch extends ToolCommand {
filterSameNames: Boolean = true,
rowSampleRegex: Option[Regex] = None,
columnSampleRegex: Option[Regex] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('i', "input") required () unbounded () valueName "<file>" action { (x, c) =>
c.copy(inputMetrics = x)
} text "Input should be a table where the first row and column have the ID's, those can be different"
......
......@@ -14,19 +14,18 @@
*/
package nl.lumc.sasc.biopet.tools
import java.io.{PrintWriter, File}
import java.io.{File, PrintWriter}
import htsjdk.samtools.{QueryInterval, SAMRecord, SamReaderFactory, ValidationStringency}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.JavaConversions._
import scala.io.Source
object FindRepeatsPacBio extends ToolCommand {
case class Args(inputBam: File = null, outputFile: Option[File] = None, inputBed: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputBam") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(inputBam = x)
} text "Path to input file"
......
......@@ -24,7 +24,7 @@ import htsjdk.variant.variantcontext.writer.{
}
import htsjdk.variant.variantcontext.{Allele, GenotypeBuilder, VariantContextBuilder}
import htsjdk.variant.vcf._
import nl.lumc.sasc.biopet.utils.{FastaUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, FastaUtils, ToolCommand}
import scala.collection.JavaConversions._
import scala.io.Source
......@@ -41,9 +41,8 @@ object GensToVcf extends ToolCommand {
referenceFasta: File = null,
contig: String = null,
sortInput: Boolean = false)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('g', "inputGenotypes") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(inputGenotypes = x)
} text "Input genotypes"
......
package nl.lumc.sasc.biopet.tools
import nl.lumc.sasc.biopet.utils.{FastaUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, FastaUtils, ToolCommand}
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.annotation.{Exon, Feature, Gene, Transcript}
......@@ -14,9 +14,8 @@ import scala.io.Source
object GtfToRefflat extends ToolCommand {
case class Args(refFlat: File = null, gtfFile: File = null, referenceFasta: Option[File] = None)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('r', "refFlat") required () unbounded () valueName "<file>" action { (x, c) =>
c.copy(refFlat = x)
} text "Input refFlat file. Mandatory"
......
......@@ -18,7 +18,7 @@ import java.io.{File, PrintWriter}
import htsjdk.variant.variantcontext.VariantContext
import htsjdk.variant.vcf.VCFFileReader
import nl.lumc.sasc.biopet.utils.{VcfUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand, VcfUtils}
import nl.lumc.sasc.biopet.utils.intervals.BedRecord
import scala.collection.JavaConversions._
......@@ -35,9 +35,8 @@ object GvcfToBed extends ToolCommand {
sample: Option[String] = None,
minGenomeQuality: Int = 0,
inverse: Boolean = false)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputVcf") required () maxOccurs 1 valueName "<file>" action { (x, c) =>
c.copy(inputVcf = x)
} text "Input vcf file"
......
......@@ -20,7 +20,7 @@ package nl.lumc.sasc.biopet.tools
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.ConfigUtils._
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
......@@ -60,9 +60,8 @@ object KrakenReportToJson extends ToolCommand {
case class Args(krakenreport: File = null,
outputJson: Option[File] = None,
skipNames: Boolean = false)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
head(s"""
|$commandName - Convert Kraken-report (full) output to JSON
......
......@@ -23,7 +23,7 @@ import htsjdk.variant.variantcontext.writer.{
}
import htsjdk.variant.variantcontext.{Allele, VariantContext, VariantContextBuilder}
import htsjdk.variant.vcf.{VCFFileReader, VCFHeader}
import nl.lumc.sasc.biopet.utils.{FastaUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, FastaUtils, ToolCommand}
import nl.lumc.sasc.biopet.utils.config.Configurable
import scala.collection.JavaConversions._
......@@ -32,9 +32,8 @@ import scala.collection.{SortedMap, mutable}
object MergeAlleles extends ToolCommand {
case class Args(inputFiles: List[File] = Nil, outputFile: File = null, reference: File = null)
extends AbstractArgs
class OptParser extends AbstractOptParser {
class OptParser extends AbstractOptParser[Args](commandName) {
opt[File]('I', "inputVcf") minOccurs 2 required () unbounded () valueName "<file>" action {
(x, c) =>
c.copy(inputFiles = x :: c.inputFiles)
......
......@@ -14,9 +14,9 @@
*/
package nl.lumc.sasc.biopet.tools
import java.io.{PrintWriter, File}
import java.io.{File, PrintWriter}
import nl.lumc.sasc.biopet.utils.ToolCommand
import nl.lumc.sasc.biopet.utils.{AbstractOptParser, ToolCommand}
import scala.io.Source
......@@ -27,9 +27,8 @@ object MergeOtuMaps extends ToolCommand {
case class Args(inputFiles: List[File] = Nil,
outputFile: File = null,
skipPrefix: List[String] = Nil)