Merge remote-tracking branch 'remotes/origin/develop' into fix-BIOPET-750

# Conflicts:
#	biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FastqSync.scala

bammetrics/src/main/scala/nl/lumc/sasc/biopet/pipelines/bammetrics/BammetricsReport.scala

@@ -284,6 +284,9 @@ object BammetricsReport extends ReportBuilder {
           .getOrElse(throw new IllegalStateException("Sample must be there"))
         val libraryName =
           library.flatMap(l => Await.result(summary.getLibraryName(l), Duration.Inf))
+        if (yKeyList.find(x => map.contains(x) && map(x).isDefined).isEmpty) {
+          ""
+        }
         val yKey = yKeyList.find(x => map.contains(x) && map(x).isDefined).get
         val xKey = xKeyList.find(x => map.contains(x) && map(x).isDefined).get
         Map(

biopet-core/src/main/scala/nl/lumc/sasc/biopet/extensions/tools/FastqSync.scala

@@ -16,7 +16,7 @@ package nl.lumc.sasc.biopet.extensions.tools
 
 import java.io.File
 
-import nl.lumc.sasc.biopet.core.{BiopetCommandLineFunction, ToolCommandFunction}
+import nl.lumc.sasc.biopet.core.ToolCommandFunction
 import nl.lumc.sasc.biopet.core.summary.Summarizable
 import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.utils.commandline.{Input, Output}
@@ -35,7 +35,11 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa
 
   /** Original FASTQ file (read 1 or 2) */
   @Input(required = true)
-  var refFastq: File = _
+  var refFastq1: File = _
+
+  /** Original FASTQ file (read 1 or 2) */
+  @Input(required = true)
+  var refFastq2: File = _
 
   /** "Input read 1 FASTQ file" */
   @Input(required = true)
@@ -61,7 +65,8 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa
 
   override def cmdLine: String =
     super.cmdLine +
-      required("-r", refFastq) +
+      required("-r", refFastq1) +
+      required("--ref2", refFastq2) +
       required("-i", inputFastq1) +
       required("-j", inputFastq2) +
       required("-o", outputFastq1) +

biopet-tools/src/main/scala/nl/lumc/sasc/biopet/tools/FastqSync.scala

@@ -24,12 +24,13 @@ import scala.collection.JavaConverters._
 
 object FastqSync extends ToolCommand {
 
-  /** Regex for capturing read ID ~ taking into account its read pair mark (if present) */
-  private val idRegex = """[_/][12]$""".r
-
   /** Implicit class to allow for lazy retrieval of FastqRecord ID without any read pair mark */
   private implicit class FastqPair(fq: FastqRecord) {
-    lazy val fragId: String = idRegex.split(fq.getReadHeader.split(" ")(0))(0)
+    lazy val fragId: String = fq.getReadHeader.split(" ").head match {
+      case x if x.endsWith(idSufixes._1) => x.stripSuffix(idSufixes._1)
+      case x if x.endsWith(idSufixes._2) => x.stripSuffix(idSufixes._2)
+      case x => x
+    }
   }
 
   /**
@@ -106,11 +107,12 @@ object FastqSync extends ToolCommand {
     (numDiscA, numDiscB, numKept)
   }
 
-  case class Args(refFastq: File = new File(""),
-                  inputFastq1: File = new File(""),
-                  inputFastq2: File = new File(""),
-                  outputFastq1: File = new File(""),
-                  outputFastq2: File = new File(""))
+  case class Args(refFastq1: File = null,
+                  refFastq2: File = null,
+                  inputFastq1: File = null,
+                  inputFastq2: File = null,
+                  outputFastq1: File = null,
+                  outputFastq2: File = null)
 
   class OptParser extends AbstractOptParser[Args](commandName) {
 
@@ -121,29 +123,35 @@ object FastqSync extends ToolCommand {
         |file will be gzipped when the input is also gzipped.
       """.stripMargin)
 
-    opt[File]('r', "ref") required () valueName "<fastq>" action { (x, c) =>
-      c.copy(refFastq = x)
+    opt[File]('r', "ref1") unbounded () required () valueName "<fastq>" action { (x, c) =>
+      c.copy(refFastq1 = x)
     } validate { x =>
       if (x.exists) success else failure("Reference FASTQ file not found")
-    } text "Reference FASTQ file"
+    } text "Reference R1 FASTQ file"
 
-    opt[File]('i', "in1") required () valueName "<fastq>" action { (x, c) =>
+    opt[File]("ref2") unbounded () required () valueName "<fastq>" action { (x, c) =>
+      c.copy(refFastq2 = x)
+    } validate { x =>
+      if (x.exists) success else failure("Reference FASTQ file not found")
+    } text "Reference R2 FASTQ file"
+
+    opt[File]('i', "in1") unbounded () required () valueName "<fastq>" action { (x, c) =>
       c.copy(inputFastq1 = x)
     } validate { x =>
       if (x.exists) success else failure("Input FASTQ file 1 not found")
     } text "Input FASTQ file 1"
 
-    opt[File]('j', "in2") required () valueName "<fastq[.gz]>" action { (x, c) =>
+    opt[File]('j', "in2") unbounded () required () valueName "<fastq[.gz]>" action { (x, c) =>
       c.copy(inputFastq2 = x)
     } validate { x =>
       if (x.exists) success else failure("Input FASTQ file 2 not found")
     } text "Input FASTQ file 2"
 
-    opt[File]('o', "out1") required () valueName "<fastq[.gz]>" action { (x, c) =>
+    opt[File]('o', "out1") unbounded () required () valueName "<fastq[.gz]>" action { (x, c) =>
       c.copy(outputFastq1 = x)
     } text "Output FASTQ file 1"
 
-    opt[File]('p', "out2") required () valueName "<fastq>" action { (x, c) =>
+    opt[File]('p', "out2") unbounded () required () valueName "<fastq>" action { (x, c) =>
       c.copy(outputFastq2 = x)
     } text "Output FASTQ file 2"
   }
@@ -163,7 +171,9 @@ object FastqSync extends ToolCommand {
 
     val commandArgs: Args = parseArgs(args)
 
-    val refReader = new FastqReader(commandArgs.refFastq)
+    idSufixes = findR1R2Suffixes(commandArgs.refFastq1, commandArgs.refFastq2)
+
+    val refReader = new FastqReader(commandArgs.refFastq1)
     val AReader = new FastqReader(commandArgs.inputFastq1)
     val BReader = new FastqReader(commandArgs.inputFastq2)
     val AWriter = new AsyncFastqWriter(new BasicFastqWriter(commandArgs.outputFastq1), 3000)
@@ -182,4 +192,28 @@ object FastqSync extends ToolCommand {
       BWriter.close()
     }
   }
+
+  /**
+    * This method will look up the unique suffix for R1 and R2
+    *
+    * @param fastqR1 input R1 file
+    * @param fastqR2 Input R2 file
+    * @return suffix for (R1, R2)
+    */
+  def findR1R2Suffixes(fastqR1: File, fastqR2: File): (String, String) = {
+    val refReader1 = new FastqReader(fastqR1)
+    val refReader2 = new FastqReader(fastqR2)
+    val r1Name = refReader1.next().getReadHeader.split(" ").head
+    val r2Name = refReader2.next().getReadHeader.split(" ").head
+    refReader1.close()
+    refReader2.close()
+
+    val genericName = new String(r1Name.zip(r2Name).takeWhile(x => x._1 == x._2).map(_._1).toArray)
+
+    (r1Name.stripPrefix(genericName), r2Name.stripPrefix(genericName))
+  }
+
+  /** Regex for capturing read ID ~ taking into account its read pair mark (if present) */
+  private[tools] var idSufixes: (String, String) = _
+
 }

biopet-tools/src/test/scala/nl/lumc/sasc/biopet/tools/FastqSyncTest.scala

@@ -59,6 +59,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                   bMock: FastqReader,
                   aOutMock: AsyncFastqWriter,
                   bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver("1", "2", "3")
@@ -105,6 +106,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                     bMock: FastqReader,
                     aOutMock: AsyncFastqWriter,
                     bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver()
@@ -123,6 +125,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                     bMock: FastqReader,
                     aOutMock: AsyncFastqWriter,
                     bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver("1", "2", "3")
@@ -141,6 +144,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                       bMock: FastqReader,
                       aOutMock: AsyncFastqWriter,
                       bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver("2", "3")
@@ -168,6 +172,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                       bMock: FastqReader,
                       aOutMock: AsyncFastqWriter,
                       bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver("1", "2", "3")
@@ -195,6 +200,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                        bMock: FastqReader,
                        aOutMock: AsyncFastqWriter,
                        bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1", "2", "3")
     when(aMock.iterator) thenReturn recordsOver("2", "3")
@@ -245,6 +251,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                                     bMock: FastqReader,
                                     aOutMock: AsyncFastqWriter,
                                     bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1/1", "2/1", "3/1")
     when(aMock.iterator) thenReturn recordsOver("2/1", "3/1")
@@ -267,6 +274,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                                          bMock: FastqReader,
                                          aOutMock: AsyncFastqWriter,
                                          bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("_1", "_2")
 
     when(refMock.iterator) thenReturn recordsOver("1_1", "2_1", "3_1")
     when(aMock.iterator) thenReturn recordsOver("2_1", "3_1")
@@ -289,6 +297,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                                       bMock: FastqReader,
                                       aOutMock: AsyncFastqWriter,
                                       bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1 desc1b", "2 desc2b", "3 desc3b")
     when(aMock.iterator) thenReturn recordsOver("2 desc2a", "3 desc3a")
@@ -311,6 +320,7 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
                   bMock: FastqReader,
                   aOutMock: AsyncFastqWriter,
                   bOutMock: AsyncFastqWriter): Unit = {
+    FastqSync.idSufixes = ("/1", "/2")
 
     when(refMock.iterator) thenReturn recordsOver("1/2 yep",
                                                   "2/2 yep",
@@ -338,6 +348,8 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
     val args = Array(
       "-r",
       resourcePath("/paired01a.fq"),
+      "--ref2",
+      resourcePath("/paired01b.fq"),
       "-i",
       resourcePath("/paired01a.fq"),
       "-j",
@@ -348,10 +360,34 @@ class FastqSyncTest extends TestNGSuite with MockitoSugar with Matchers {
       "/tmp/mockout2.fq"
     )
     val parsed = parseArgs(args)
-    parsed.refFastq shouldBe resourceFile("/paired01a.fq")
+    parsed.refFastq1 shouldBe resourceFile("/paired01a.fq")
     parsed.inputFastq1 shouldBe resourceFile("/paired01a.fq")
     parsed.inputFastq2 shouldBe resourceFile("/paired01b.fq")
     parsed.outputFastq1 shouldBe new File("/tmp/mockout1.fq")
     parsed.outputFastq2 shouldBe new File("/tmp/mockout2.fq")
   }
+
+  @Test
+  def testMain: Unit = {
+    val r1Output = File.createTempFile("temp.", ".fq")
+    r1Output.deleteOnExit()
+    val r2Output = File.createTempFile("temp.", ".fq")
+    r2Output.deleteOnExit()
+    val args = Array(
+      "-r",
+      resourcePath("/paired01a.fq"),
+      "--ref2",
+      resourcePath("/paired01b.fq"),
+      "-i",
+      resourcePath("/paired01a.fq"),
+      "-j",
+      resourcePath("/paired01b.fq"),
+      "-o",
+      r1Output.getAbsolutePath,
+      "-p",
+      r2Output.getAbsolutePath
+    )
+
+    FastqSync.main(args)
+  }
 }

flexiprep/src/main/scala/nl/lumc/sasc/biopet/pipelines/flexiprep/Flexiprep.scala

@@ -208,7 +208,8 @@ class Flexiprep(val parent: Configurable)
       qcCmdR2.compress = false
 
       val fqSync = new FastqSync(this)
-      fqSync.refFastq = R1_in
+      fqSync.refFastq1 = R1_in
+      fqSync.refFastq2 = R2_in.get
       fqSync.inputFastq1 = qcCmdR1.output
       fqSync.inputFastq2 = qcCmdR2.output
       fqSync.outputFastq1 = new File(outDir, fastqR1Qc.getName)

mapping/src/main/scala/nl/lumc/sasc/biopet/pipelines/mapping/MultisampleMapping.scala

@@ -120,8 +120,8 @@ trait MultisampleMappingTrait extends MultiSampleQScript with Reference { qscrip
 
       lazy val inputR1: Option[File] = MultisampleMapping.fileMustBeAbsolute(config("R1"))
       lazy val inputR2: Option[File] = MultisampleMapping.fileMustBeAbsolute(config("R2"))
-      lazy val qcFastqR1 = mapping.map(_.flexiprep.fastqR1Qc)
-      lazy val qcFastqR2 = mapping.flatMap(_.flexiprep.fastqR2Qc)
+      lazy val qcFastqR1: Option[File] = mapping.map(_.flexiprep.fastqR1Qc)
+      lazy val qcFastqR2: Option[File] = mapping.flatMap(_.flexiprep.fastqR2Qc)
       lazy val inputBam: Option[File] =
         MultisampleMapping.fileMustBeAbsolute(if (inputR1.isEmpty) config("bam") else None)
       lazy val bamToFastq: Boolean = config("bam_to_fastq", default = false)
@@ -229,17 +229,19 @@ trait MultisampleMappingTrait extends MultiSampleQScript with Reference { qscrip
                 qscript.add(aorrg)
               }
             } else add(Ln.linkBamFile(qscript, inputBam.get, bamFile.get): _*)
+          }
 
+          if (!bamToFastq) {
             val bamMetrics = new BamMetrics(qscript)
             bamMetrics.sampleId = Some(sampleId)
             bamMetrics.libId = Some(libId)
             bamMetrics.inputBam = bamFile.get
             bamMetrics.outputDir = new File(libDir, "metrics")
-            bamMetrics.paired = inputR2.isDefined
+            bamMetrics.paired = inputR2.isDefined || inputBam.isDefined
             add(bamMetrics)
-
-            if (config("execute_bam2wig", default = true)) add(Bam2Wig(qscript, bamFile.get))
           }
+          if (config("execute_bam2wig", default = true)) add(Bam2Wig(qscript, bamFile.get))
+
         } else logger.warn(s"Sample '$sampleId' does not have any input files")
       }
     }
@@ -340,7 +342,8 @@ trait MultisampleMappingTrait extends MultiSampleQScript with Reference { qscrip
         bamMetrics.sampleId = Some(sampleId)
         bamMetrics.inputBam = if (metricsPreprogressBam) preProcessBam.get else bamFile.get
         bamMetrics.outputDir = new File(sampleDir, "metrics")
-        bamMetrics.paired = libraries.exists(_._2.inputR1.isDefined)
+        bamMetrics.paired =
+          libraries.exists(x => x._2.inputR1.isDefined || x._2.inputBam.isDefined)
         add(bamMetrics)
 
         if (config("execute_bam2wig", default = true)) add(Bam2Wig(qscript, preProcessBam.get))