Merge pull request #47 from biopet/fix-BIOPET-575

Added a exon flagstat

Jenkinsfile

 node('local') {
-    try {
+    timeout(45) {
+        try {
 
-        stage('Init') {
-            tool 'JDK 8u102'
-            tool 'Apache Maven 3.3.9'
-        }
+            stage('Init') {
+                tool 'JDK 8u102'
+                tool 'Apache Maven 3.3.9'
+            }
 
-        stage('Checkout') {
-            checkout scm
-            sh 'git submodule update --init --recursive'
-        }
+            stage('Checkout') {
+                checkout scm
+                sh 'git submodule update --init --recursive'
+            }
 
-        stage('Build and Test') {
-            withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
-                sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
+            stage('Build and Test') {
+                withMaven(maven: 'Apache Maven 3.3.9', jdk: 'JDK 8u102') {
+                    sh 'mvn -B -T 2 -Dmaven.test.failure.ignore clean package'
+                }
             }
-        }
 
-        stage('Report Tests') {
-            junit '*/target/surefire-reports/*.xml'
-        }
+            stage('Report Tests') {
+                junit '*/target/surefire-reports/*.xml'
+            }
 
-        stage('Check Documentation') {
-            sh 'mkdocs build --clean --strict'
-        }
+            stage('Check git on changes') {
+                sh 'if [ $(git diff | wc -l) -eq 0 ]; then true; else echo "[ERROR] Git is not clean anymore after build"; git diff; echo "[ERROR] This might be caused by reformated code, if so run maven locally"; false; fi'
+            }
 
-        if(currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
-            currentBuild.result = "SUCCESS"
-            slackSend (color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
-        } else {
-            slackSend (color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
-        }
+            stage('Check Documentation') {
+                sh 'mkdocs build --clean --strict'
+            }
 
-    } catch (e) {
-        if(currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
-            currentBuild.result = "FAILED"
-        }
-        slackSend (color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+            if (currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
+                currentBuild.result = "SUCCESS"
+                slackSend(color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+            } else {
+                slackSend(color: '#FFFF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
+            }
+
+        } catch (e) {
+            if (currentBuild.result == null || "FAILED".equals(currentBuild.result)) {
+                currentBuild.result = "FAILED"
+            }
+            slackSend(color: '#FF0000', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')
 
-        throw e
+            throw e
+        }
     }
 }

README.md

@@ -5,7 +5,7 @@
 
 Biopet (Bio Pipeline Execution Toolkit) is the main pipeline development framework of the LUMC Sequencing Analysis Support Core team. It contains our main pipelines and some of the command line tools we develop in-house. It is meant to be used in the main [SHARK](https://humgenprojects.lumc.nl/trac/shark) computing cluster. While usage outside of SHARK is technically possible, some adjustments may need to be made in order to do so.
 
-Full documantation is here: [Biopet documantation](http://biopet-docs.readthedocs.io/en/latest/)
+Full documentation is here: [Biopet documentation](http://biopet-docs.readthedocs.io/en/latest/)
 
 ## Quick Start
 
@@ -60,7 +60,7 @@ Biopet is based on the Queue framework developed by the Broad Institute as part
 
 We welcome any kind of contribution, be it merge requests on the code base, documentation updates, or any kinds of other fixes! The main language we use is Scala, though the repository also contains a small bit of Python and R. Our main code repository is located at [https://github.com/biopet/biopet](https://github.com/biopet/biopet/issues), along with our issue tracker.
 
-For more information please go to our [Developer documantation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/)
+For more information please go to our [Developer documentation](http://biopet-docs.readthedocs.io/en/develop/developer/getting-started/)
 
 ## About
 

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/report/ReportBuilder.scala

@@ -20,7 +20,7 @@ import nl.lumc.sasc.biopet.core.ToolCommandFunction
 import nl.lumc.sasc.biopet.utils.summary.Summary
 import nl.lumc.sasc.biopet.utils.{ IoUtils, Logging, ToolCommand }
 import org.broadinstitute.gatk.utils.commandline.Input
-import org.fusesource.scalate.{ TemplateEngine, TemplateSource }
+import org.fusesource.scalate.TemplateEngine
 
 import scala.collection.mutable
 import scala.language.postfixOps
@@ -236,9 +236,7 @@ object ReportBuilder {
 
   /** Single template render engine, this will have a cache for all compile templates */
   protected val engine = new TemplateEngine()
-
-  /** Cache of temp file for templates from the classpath / jar */
-  private[report] var templateCache: Map[String, File] = Map()
+  engine.allowReload = false
 
   /** This will give the total number of pages including all nested pages */
   def countPages(page: ReportPage): Int = {
@@ -254,15 +252,6 @@ object ReportBuilder {
   def renderTemplate(location: String, args: Map[String, Any] = Map()): String = {
     Logging.logger.info("Rendering: " + location)
 
-    val templateFile: File = templateCache.get(location) match {
-      case Some(template) => template
-      case _ =>
-        val tempFile = File.createTempFile("ssp-template", new File(location).getName)
-        tempFile.deleteOnExit()
-        IoUtils.copyStreamToFile(getClass.getResourceAsStream(location), tempFile)
-        templateCache += location -> tempFile
-        tempFile
-    }
-    engine.layout(TemplateSource.fromFile(templateFile), args)
+    engine.layout(location, args)
   }
 }
\ No newline at end of file

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/report/ReportBuilderTest.scala

@@ -79,11 +79,8 @@ class ReportBuilderTest extends TestNGSuite with Matchers {
 
   @Test
   def testRenderTemplate: Unit = {
-    ReportBuilder.templateCache = Map()
-    ReportBuilder.templateCache shouldBe empty
     ReportBuilder.renderTemplate("/template.ssp", Map("arg" -> "test")) shouldBe "test"
-    ReportBuilder.templateCache.size shouldBe 1
     ReportBuilder.renderTemplate("/template.ssp", Map("arg" -> "bla")) shouldBe "bla"
-    ReportBuilder.templateCache.size shouldBe 1
   }
+
 }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Fastqc.scala

@@ -75,8 +75,8 @@ class Fastqc(val root: Configurable) extends BiopetCommandLineFunction with Vers
       // otherwise, check if adapters are already present (depending on FastQC version)
       case None =>
         val defaultAdapters = getVersion match {
-          case Some("v0.11.2") => Option(new File(fastqcDir + "/Configuration/adapter_list.txt"))
-          case _               => None
+          case Some(v) if v.contains("v0.11") => Option(new File(fastqcDir + "/Configuration/adapter_list.txt"))
+          case _                              => None
         }
         defaultAdapters.collect { case adp => config("adapters", default = adp) }
     }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/delly/DellyCaller.scala

@@ -42,8 +42,19 @@ class DellyCaller(val root: Configurable) extends BiopetCommandLineFunction with
   var analysistype: String = _
 
   def cmdLine = required(executable) +
-    "-t" + required(analysistype) +
-    "-o" + required(outputvcf) +
-    required(input)
+    required("-t", analysistype) +
+    required("-o", outputvcf) +
+    required(input) +
+    createEmptyOutputIfNeeded
+
+  // when no variants are found then the tool doesn't generate the output file either, in Biopet it's needed that the empty file would be there
+  private def createEmptyOutputIfNeeded =
+    s"""
+       |c=$$?
+       |if [ $$c -eq 0 ] && [ ! -f $outputvcf ]; then
+       |  echo '##fileformat=VCFv4.2' > $outputvcf
+       |  echo '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO' >> $outputvcf
+       |fi
+       |exit $$c""".stripMargin
 
 }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/gatk/CatVariants.scala

@@ -37,6 +37,9 @@ class CatVariants(val root: Configurable) extends BiopetJavaCommandLineFunction
   @Gather(classOf[org.broadinstitute.gatk.queue.function.scattergather.SimpleTextGatherFunction])
   var outputFile: File = _
 
+  /** When Gatk's CatVariants has been called with empty VCF-files, then it also outputs an empty file. When this parameter is set to true, then the empty file gets added a valid VCF-file header.*/
+  var writeHeaderToEmptyOutput: Boolean = false
+
   /** assumeSorted should be true if the input files are already sorted (based on the position of the variants) */
   @Argument(fullName = "assumeSorted", shortName = "assumeSorted", doc = "assumeSorted should be true if the input files are already sorted (based on the position of the variants)", required = false, exclusiveOf = "", validation = "")
   var assumeSorted: Boolean = _
@@ -73,5 +76,14 @@ class CatVariants(val root: Configurable) extends BiopetJavaCommandLineFunction
     optional("--variant_index_type", variant_index_type, spaceSeparated = true, escape = true, format = "%s") +
     optional("--variant_index_parameter", variant_index_parameter, spaceSeparated = true, escape = true, format = "%s") +
     optional("-l", logging_level, spaceSeparated = true, escape = true, format = "%s") +
-    optional("-log", log_to_file, spaceSeparated = true, escape = true, format = "%s")
+    optional("-log", log_to_file, spaceSeparated = true, escape = true, format = "%s") +
+    (if (writeHeaderToEmptyOutput) s"""
+       |c=$$?
+       |if [ $$c -eq 0 ] && [ ! -s $outputFile ]; then
+       |  echo '##fileformat=VCFv4.2' > $outputFile
+       |  echo '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO' >> $outputFile
+       |fi
+       |exit $$c""".stripMargin
+    else "")
+
 }

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/samtools/SamtoolsFlagstat.scala

@@ -28,7 +28,9 @@ class SamtoolsFlagstat(val root: Configurable) extends Samtools {
   var output: File = _
 
   /** Returns command to execute */
-  def cmdLine = required(executable) + required("flagstat") + required(input) + " > " + required(output)
+  def cmdLine = required(executable) + required("flagstat") +
+    (if (inputAsStdin) "-" else required(input)) +
+    (if (outputAsStsout) "" else " > " + required(output))
 }
 
 object SamtoolsFlagstat {

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/samtools/SamtoolsView.scala

@@ -33,18 +33,19 @@ class SamtoolsView(val root: Configurable) extends Samtools {
   var f: List[String] = config("f", default = List.empty[String])
   var F: List[String] = config("F", default = List.empty[String])
 
-  def cmdBase = required(executable) +
+  @Input(required = false)
+  var L: Option[File] = None
+
+  def cmdLine = required(executable) +
     required("view") +
     optional("-q", q) +
+    optional("-L", L) +
     repeat("-f", f) +
     repeat("-F", F) +
     conditional(b, "-b") +
-    conditional(h, "-h")
-  def cmdPipeInput = cmdBase + "-"
-  def cmdPipe = cmdBase + required(input)
-
-  /** Returns command to execute */
-  def cmdLine = cmdPipe + " > " + required(output)
+    conditional(h, "-h") +
+    (if (inputAsStdin) "-" else required(input)) +
+    (if (outputAsStsout) "" else " > " + required(output))
 }
 
 object SamtoolsView {

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/bamstats/BamStats.scala

@@ -39,7 +39,8 @@ object BamStats extends ToolCommand {
                   bamFile: File = null,
                   referenceFasta: Option[File] = None,
                   binSize: Int = 10000,
-                  threadBinSize: Int = 10000000) extends AbstractArgs
+                  threadBinSize: Int = 10000000,
+                  tsvOutputs: Boolean = false) extends AbstractArgs
 
   class OptParser extends AbstractOptParser {
     opt[File]('R', "reference") valueName "<file>" action { (x, c) =>
@@ -57,6 +58,9 @@ object BamStats extends ToolCommand {
     opt[Int]("threadBinSize") valueName "<int>" action { (x, c) =>
       c.copy(threadBinSize = x)
     } text "Size of region per thread"
+    opt[Unit]("tsvOutputs") action { (x, c) =>
+      c.copy(tsvOutputs = true)
+    } text "Also output tsv files, default there is only a json"
   }
 
   /** This is the main entry to [[BamStats]], this will do the argument parsing. */
@@ -68,7 +72,7 @@ object BamStats extends ToolCommand {
 
     val sequenceDict = validateReferenceInBam(cmdArgs.bamFile, cmdArgs.referenceFasta)
 
-    init(cmdArgs.outputDir, cmdArgs.bamFile, sequenceDict, cmdArgs.binSize, cmdArgs.threadBinSize)
+    init(cmdArgs.outputDir, cmdArgs.bamFile, sequenceDict, cmdArgs.binSize, cmdArgs.threadBinSize, cmdArgs.tsvOutputs)
 
     logger.info("Done")
   }
@@ -96,13 +100,26 @@ object BamStats extends ToolCommand {
    * @param binSize stats binsize
    * @param threadBinSize Thread binsize
    */
-  def init(outputDir: File, bamFile: File, referenceDict: SAMSequenceDictionary, binSize: Int, threadBinSize: Int): Unit = {
+  def init(outputDir: File, bamFile: File, referenceDict: SAMSequenceDictionary, binSize: Int, threadBinSize: Int, tsvOutput: Boolean): Unit = {
     val contigsFutures = BedRecordList.fromDict(referenceDict).allRecords.map { contig =>
       contig.chr -> processContig(contig, bamFile, binSize, threadBinSize, outputDir)
     }.toList
 
     val stats = waitOnFutures(processUnmappedReads(bamFile) :: contigsFutures.map(_._2))
 
+    if (tsvOutput) {
+      stats.flagstat.writeAsTsv(new File(outputDir, "flagstats.tsv"))
+
+      stats.insertSizeHistogram.writeFilesAndPlot(outputDir, "insertsize", "Insertsize", "Reads", "Insertsize distribution")
+      stats.mappingQualityHistogram.writeFilesAndPlot(outputDir, "mappingQuality", "Mapping Quality", "Reads", "Mapping Quality distribution")
+      stats.clippingHistogram.writeFilesAndPlot(outputDir, "clipping", "CLipped bases", "Reads", "Clipping distribution")
+
+      stats.leftClippingHistogram.writeFilesAndPlot(outputDir, "left_clipping", "CLipped bases", "Reads", "Left Clipping distribution")
+      stats.rightClippingHistogram.writeFilesAndPlot(outputDir, "right_clipping", "CLipped bases", "Reads", "Right Clipping distribution")
+      stats._3_ClippingHistogram.writeFilesAndPlot(outputDir, "3prime_clipping", "CLipped bases", "Reads", "3 Prime Clipping distribution")
+      stats._5_ClippingHistogram.writeFilesAndPlot(outputDir, "5prime_clipping", "CLipped bases", "Reads", "5 Prime Clipping distribution")
+    }
+
     val statsWriter = new PrintWriter(new File(outputDir, "bamstats.json"))
     val totalStats = stats.toSummaryMap
     val statsMap = Map(

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/bamstats/Histogram.scala

@@ -14,8 +14,10 @@
  */
 package nl.lumc.sasc.biopet.tools.bamstats
 
-import java.io.{ File, PrintWriter }
-import nl.lumc.sasc.biopet.utils.sortAnyAny
+import java.io.{ File, IOException, PrintWriter }
+
+import nl.lumc.sasc.biopet.utils.rscript.LinePlot
+import nl.lumc.sasc.biopet.utils.{ Logging, sortAnyAny }
 
 import scala.collection.mutable
 
@@ -43,7 +45,7 @@ class Counts[T](_counts: Map[T, Long] = Map[T, Long]())(implicit ord: Ordering[T
   }
 
   /** Write histogram to a tsv/count file */
-  def writeToTsv(file: File): Unit = {
+  def writeHistogramToTsv(file: File): Unit = {
     val writer = new PrintWriter(file)
     writer.println("value\tcount")
     counts.keys.toList.sorted.foreach(x => writer.println(s"$x\t${counts(x)}"))
@@ -82,4 +84,28 @@ class Histogram[T](_counts: Map[T, Long] = Map[T, Long]())(implicit ord: Numeric
     } else Map()
   }
 
+  /** Write histogram to a tsv/count file */
+  def writeAggregateToTsv(file: File): Unit = {
+    val writer = new PrintWriter(file)
+    aggregateStats.foreach(x => writer.println(x._1 + "\t" + x._2))
+    writer.close()
+  }
+
+  def writeFilesAndPlot(outputDir: File, prefix: String, xlabel: String, ylabel: String, title: String): Unit = {
+    writeHistogramToTsv(new File(outputDir, prefix + ".histogram.tsv"))
+    writeAggregateToTsv(new File(outputDir, prefix + ".stats.tsv"))
+    val plot = new LinePlot(null)
+    plot.input = new File(outputDir, prefix + ".histogram.tsv")
+    plot.output = new File(outputDir, prefix + ".histogram.png")
+    plot.xlabel = Some(xlabel)
+    plot.ylabel = Some(ylabel)
+    plot.title = Some(title)
+    try {
+      plot.runLocal()
+    } catch {
+      // If plotting fails the tools should not fail, this depens on R to be installed
+      case e: IOException => Logging.logger.warn(s"Error found while plotting ${plot.output}: ${e.getMessage}")
+    }
+  }
+
 }

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/bamstats/Stats.scala

@@ -50,13 +50,13 @@ case class Stats(flagstat: FlagstatCollector = new FlagstatCollector(),
   def writeStatsToFiles(outputDir: File): Unit = {
     this.flagstat.writeReportToFile(new File(outputDir, "flagstats"))
     this.flagstat.writeSummaryTofile(new File(outputDir, "flagstats.summary.json"))
-    this.mappingQualityHistogram.writeToTsv(new File(outputDir, "mapping_quality.tsv"))
-    this.insertSizeHistogram.writeToTsv(new File(outputDir, "insert_size.tsv"))
-    this.clippingHistogram.writeToTsv(new File(outputDir, "clipping.tsv"))
-    this.leftClippingHistogram.writeToTsv(new File(outputDir, "left_clipping.tsv"))
-    this.rightClippingHistogram.writeToTsv(new File(outputDir, "right_clipping.tsv"))
-    this._5_ClippingHistogram.writeToTsv(new File(outputDir, "5_prime_clipping.tsv"))
-    this._3_ClippingHistogram.writeToTsv(new File(outputDir, "3_prime_clipping.tsv"))
+    this.mappingQualityHistogram.writeHistogramToTsv(new File(outputDir, "mapping_quality.tsv"))
+    this.insertSizeHistogram.writeHistogramToTsv(new File(outputDir, "insert_size.tsv"))
+    this.clippingHistogram.writeHistogramToTsv(new File(outputDir, "clipping.tsv"))
+    this.leftClippingHistogram.writeHistogramToTsv(new File(outputDir, "left_clipping.tsv"))
+    this.rightClippingHistogram.writeHistogramToTsv(new File(outputDir, "right_clipping.tsv"))
+    this._5_ClippingHistogram.writeHistogramToTsv(new File(outputDir, "5_prime_clipping.tsv"))
+    this._3_ClippingHistogram.writeHistogramToTsv(new File(outputDir, "3_prime_clipping.tsv"))
   }
 
   def toSummaryMap = {

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/flagstat/FlagstatCollector.scala

@@ -32,6 +32,12 @@ class FlagstatCollector {
   protected[FlagstatCollector] var totalCounts: Array[Long] = Array()
   protected[FlagstatCollector] var crossCounts = Array.ofDim[Long](1, 1)
 
+  def writeAsTsv(file: File): Unit = {
+    val writer = new PrintWriter(file)
+    names.foreach(x => writer.println(x._2 + "\t" + totalCounts(x._1)))
+    writer.close()
+  }
+
   def loadDefaultFunctions() {
     addFunction("All", record => true)
     addFunction("Mapped", record => !record.getReadUnmappedFlag)

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/vcfstats/VcfStats.scala

@@ -74,7 +74,7 @@ object VcfStats extends ToolCommand {
     opt[File]('o', "outputDir") required () unbounded () maxOccurs 1 valueName "<file>" action { (x, c) =>
       c.copy(outputDir = x)
     } validate {
-      x => if (x == null) failure("Output directory required") else success
+      x => if (x == null) failure("Valid output directory required") else if (x.exists) success else failure(s"Output directory does not exist: $x")
     } text "Path to directory for output (required)"
     opt[File]('i', "intervals") unbounded () valueName "<file>" action { (x, c) =>
       c.copy(intervals = Some(x))

biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/VcfStatsTest.scala

@@ -24,6 +24,7 @@ import org.scalatest.Matchers
 import org.scalatest.testng.TestNGSuite
 import org.testng.annotations.Test
 import nl.lumc.sasc.biopet.utils.sortAnyAny
+import org.apache.commons.io.FileUtils
 
 import scala.collection.mutable
 
@@ -162,6 +163,16 @@ class VcfStatsTest extends TestNGSuite with Matchers {
     valueFromTsv(i, "Sample_ID_3", "bam") should be(empty)
   }
 
+  @Test
+  def testNoExistOutputDir: Unit = {
+    val tmp = Files.createTempDirectory("vcfStats")
+    FileUtils.deleteDirectory(new File(tmp.toAbsolutePath.toString))
+    val vcf = resourcePath("/chrQ.vcf.gz")
+    val ref = resourcePath("/fake_chrQ.fa")
+
+    an[IllegalArgumentException] should be thrownBy main(Array("-I", vcf, "-R", ref, "-o", tmp.toAbsolutePath.toString))
+  }
+
   @Test
   def testMain() = {
     val tmp = Files.createTempDirectory("vcfStats")

biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/bamstats/BamStatsTest.scala

@@ -34,6 +34,43 @@ class BamStatsTest extends TestNGSuite with Matchers {
 
     new File(outputDir, "bamstats.json") should exist
     new File(outputDir, "bamstats.summary.json") should exist
+
+    new File(outputDir, "flagstats.tsv") shouldNot exist
+    new File(outputDir, "insertsize.stats.tsv") shouldNot exist
+    new File(outputDir, "insertsize.histogram.tsv") shouldNot exist
+    new File(outputDir, "mappingQuality.stats.tsv") shouldNot exist
+    new File(outputDir, "mappingQuality.histogram.tsv") shouldNot exist
+    new File(outputDir, "clipping.stats.tsv") shouldNot exist
+    new File(outputDir, "clipping.histogram.tsv") shouldNot exist
+
+    new File(outputDir, "flagstats") shouldNot exist
+    new File(outputDir, "flagstats.summary.json") shouldNot exist
+    new File(outputDir, "mapping_quality.tsv") shouldNot exist
+    new File(outputDir, "insert_size.tsv") shouldNot exist
+    new File(outputDir, "clipping.tsv") shouldNot exist
+    new File(outputDir, "left_clipping.tsv") shouldNot exist
+    new File(outputDir, "right_clipping.tsv") shouldNot exist
+    new File(outputDir, "5_prime_clipping.tsv") shouldNot exist
+    new File(outputDir, "3_prime_clipping.tsv") shouldNot exist
+  }
+
+  @Test
+  def testTsvOutputs: Unit = {
+    val outputDir = Files.createTempDir()
+    outputDir.deleteOnExit()
+    BamStats.main(Array("-b", BamStatsTest.pairedBam01.getAbsolutePath, "-o", outputDir.getAbsolutePath, "--tsvOutputs"))
+
+    new File(outputDir, "bamstats.json") should exist
+    new File(outputDir, "bamstats.summary.json") should exist
+
+    new File(outputDir, "flagstats.tsv") should exist
+    new File(outputDir, "insertsize.stats.tsv") should exist
+    new File(outputDir, "insertsize.histogram.tsv") should exist
+    new File(outputDir, "mappingQuality.stats.tsv") should exist
+    new File(outputDir, "mappingQuality.histogram.tsv") should exist
+    new File(outputDir, "clipping.stats.tsv") should exist
+    new File(outputDir, "clipping.histogram.tsv") should exist
+
     new File(outputDir, "flagstats") shouldNot exist
     new File(outputDir, "flagstats.summary.json") shouldNot exist
     new File(outputDir, "mapping_quality.tsv") shouldNot exist

biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/bamstats/CountsTest.scala

@@ -86,7 +86,7 @@ class CountsTest extends TestNGSuite with Matchers {
     val tsvFile = File.createTempFile("counts.", ".tsv")
     tsvFile.deleteOnExit()
 
-    c1.writeToTsv(tsvFile)
+    c1.writeHistogramToTsv(tsvFile)
 
     val reader = Source.fromFile(tsvFile)
     reader.getLines().toList shouldBe List("value\tcount", "1\t1", "2\t2", "3\t3")

biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/bamstats/HistogramTest.scala

@@ -64,7 +64,7 @@ class HistogramTest extends TestNGSuite with Matchers {
     val tsvFile = File.createTempFile("counts.", ".tsv")
     tsvFile.deleteOnExit()
 
-    c1.writeToTsv(tsvFile)
+    c1.writeHistogramToTsv(tsvFile)
 
     val reader = Source.fromFile(tsvFile)
     reader.getLines().toList shouldBe List("value\tcount", "1\t1", "2\t2", "3\t3")

carp/src/main/scala/nl/lumc/sasc/biopet/pipelines/carp/Carp.scala

@@ -61,6 +61,8 @@ class Carp(val root: Configurable) extends QScript with MultisampleMappingTrait
 
     override def preProcessBam = Some(createFile("filter.bam"))
 
+    override def metricsPreprogressBam = false
+
     val controls: List[String] = config("control", default = Nil)
 
     override def summarySettings = super.summarySettings ++ Map("controls" -> controls)

docs/pipelines/carp.md

@@ -101,6 +101,12 @@ While optional settings are:
 1. `aligner`: which aligner to use (`bwa` or `bowtie`)
 2. `macs2`: Here only the callpeak modus is implemented. But one can set all the options from [macs2 callpeak](https://github.com/taoliu/MACS/#call-peaks) in this settings config. Note that the config value is: `macs2_callpeak`
 
+[Gears](gears) is run automatically for the data analysed with `Carp`. There are two levels on which this can be done and this should be specified in the [config](../general/config) file:
+
+*`mapping_to_gears: unmapped` : Unmapped reads after alignment. (default)
+*`mapping_to_gears: all` : Trimmed and clipped reads from [Flexiprep](flexiprep).
+*`mapping_to_gears: none` : Disable this functionality.
+
 ## Configuration for detection of broad peaks (ATAC-seq)
 
 Carp can do broad peak-calling by using the following config: