Merge remote-tracking branch 'origin/feature-index_generation' into feature-index_generation

# Conflicts:
#	generate-indexes/src/main/scala/nl/lumc/sasc/biopet/pipelines/generateindexes/GenerateIndexes.scala

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Zcat.scala

@@ -24,7 +24,7 @@ import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
 /** Extension for zcat */
 class Zcat(val root: Configurable) extends BiopetCommandLineFunction with Version {
   @Input(doc = "Zipped file", required = true)
-  var input: List[File] = _
+  var input: List[File] = Nil
 
   @Output(doc = "Unzipped file", required = true)
   var output: File = _

biopet-package/src/main/scala/nl/lumc/sasc/biopet/BiopetExecutableMain.scala

@@ -15,6 +15,7 @@
  */
 package nl.lumc.sasc.biopet
 
+import nl.lumc.sasc.biopet.pipelines.generateindexes.GenerateIndexes
 import nl.lumc.sasc.biopet.utils.{ BiopetExecutable, MainCommand }
 
 object BiopetExecutableMain extends BiopetExecutable {
@@ -36,7 +37,7 @@ object BiopetExecutableMain extends BiopetExecutable {
     nl.lumc.sasc.biopet.pipelines.shiva.ShivaVariantcalling,
     nl.lumc.sasc.biopet.pipelines.basty.Basty,
     nl.lumc.sasc.biopet.pipelines.shiva.Shiva,
-    nl.lumc.sasc.biopet.pipelines.GenerateIndexes
+    GenerateIndexes
   )
 
   def tools: List[MainCommand] = BiopetToolsExecutable.tools

generate-indexes/pom.xml

@@ -45,6 +45,18 @@
             <artifactId>BiopetExtensions</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <version>6.8</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.10</artifactId>
+            <version>2.2.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file

generate-indexes/src/main/scala/nl/lumc/sasc/biopet/pipelines/generateindexes/FastaMerging.scala

+package nl.lumc.sasc.biopet.pipelines.generateindexes
+
+import java.io.File
+
+import nl.lumc.sasc.biopet.core.BiopetCommandLineFunction
+import nl.lumc.sasc.biopet.utils.config.Configurable
+import org.broadinstitute.gatk.utils.commandline.{ Input, Output }
+
+/**
+ * Created by pjvan_thof on 13-5-16.
+ */
+class FastaMerging(val root: Configurable) extends BiopetCommandLineFunction {
+  @Input
+  var input: List[File] = Nil
+
+  @Output(required = true)
+  var output: File = _
+
+  var cmds: Array[BiopetCommandLineFunction] = Array()
+
+  def cmdLine = cmds.map(_.commandLine).mkString(" && ")
+
+}

generate-indexes/src/main/scala/nl/lumc/sasc/biopet/pipelines/GenerateIndexes.scala → generate-indexes/src/main/scala/nl/lumc/sasc/biopet/pipelines/generateindexes/GenerateIndexes.scala

@@ -13,14 +13,13 @@
  * license; For commercial users or users who do not want to follow the AGPL
  * license, please contact us to obtain a separate license.
  */
-package nl.lumc.sasc.biopet.pipelines
+package nl.lumc.sasc.biopet.pipelines.generateindexes
 
 import java.io.PrintWriter
 import java.util
 
 import nl.lumc.sasc.biopet.core.extensions.Md5sum
-import nl.lumc.sasc.biopet.utils.config.Configurable
-import nl.lumc.sasc.biopet.core.{ BiopetCommandLineFunction, BiopetQScript, PipelineCommand }
+import nl.lumc.sasc.biopet.core.{ BiopetQScript, PipelineCommand }
 import nl.lumc.sasc.biopet.extensions._
 import nl.lumc.sasc.biopet.extensions.bowtie.{ Bowtie2Build, BowtieBuild }
 import nl.lumc.sasc.biopet.extensions.bwa.BwaIndex
@@ -29,18 +28,19 @@ import nl.lumc.sasc.biopet.extensions.gmap.GmapBuild
 import nl.lumc.sasc.biopet.extensions.picard.CreateSequenceDictionary
 import nl.lumc.sasc.biopet.extensions.samtools.SamtoolsFaidx
 import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.config.Configurable
 import org.broadinstitute.gatk.queue.QScript
 
-import scala.language.reflectiveCalls
 import scala.collection.JavaConversions._
+import scala.language.reflectiveCalls
 
 class GenerateIndexes(val root: Configurable) extends QScript with BiopetQScript {
   def this() = this(null)
 
-  @Argument
-  var referenceConfigFile: File = _
+  @Argument(required = true)
+  var referenceConfigFiles: List[File] = Nil
 
-  var referenceConfig: Map[String, Any] = Map()
+  var referenceConfig: Map[String, Any] = null
 
   var configDeps: List[File] = Nil
 
@@ -48,7 +48,8 @@ class GenerateIndexes(val root: Configurable) extends QScript with BiopetQScript
 
   /** This is executed before the script starts */
   def init(): Unit = {
-    referenceConfig = ConfigUtils.fileToConfigMap(referenceConfigFile)
+    if (referenceConfig == null)
+      referenceConfig = referenceConfigFiles.foldLeft(Map[String, Any]())((a, b) => ConfigUtils.mergeMaps(a, ConfigUtils.fileToConfigMap(b)))
   }
 
   /** Method where jobs must be added */
@@ -61,8 +62,9 @@ class GenerateIndexes(val root: Configurable) extends QScript with BiopetQScript
         val genomeConfig = ConfigUtils.any2map(c)
         val fastaUris = genomeConfig.getOrElse("fasta_uri",
           throw new IllegalArgumentException(s"No fasta_uri found for $speciesName - $genomeName")) match {
-            case a: Array[_] => a.map(_.toString)
-            case a           => Array(a.toString)
+            case a: Traversable[_] => a.map(_.toString).toArray
+            case a: util.ArrayList[_] => a.map(_.toString).toArray
+            case a                 => Array(a.toString)
           }
 
         val genomeDir = new File(speciesDir, genomeName)
@@ -83,16 +85,8 @@ class GenerateIndexes(val root: Configurable) extends QScript with BiopetQScript
           curl.output
         }
 
-        val fastaCat = new CommandLineFunction {
-          var cmds: Array[BiopetCommandLineFunction] = Array()
-
-          @Input
-          var input: List[File] = Nil
-
-          @Output
-          var output = fastaFile
-          def commandLine = cmds.mkString(" && ")
-        }
+        val fastaCat = new FastaMerging(this)
+        fastaCat.output = fastaFile
 
         if (fastaUris.length > 1 || fastaFiles.exists(_.getName.endsWith(".gz"))) {
           fastaFiles.foreach { file =>
@@ -259,6 +253,8 @@ class GenerateIndexes(val root: Configurable) extends QScript with BiopetQScript
     add(new InProcessFunction {
       @Input val deps: List[File] = configDeps
 
+      @Output val out = outputConfigFile
+
       def run: Unit = {
         val writer = new PrintWriter(outputConfigFile)
         writer.println(ConfigUtils.mapToJson(Map("references" -> outputConfig)).spaces2)

generate-indexes/src/test/scala/nl/lumc/sasc/biopet/pipelines/generateindexes/GenerateIndexesTest.scala

+package nl.lumc.sasc.biopet.pipelines.generateindexes
+
+import com.google.common.io.Files
+import nl.lumc.sasc.biopet.utils.ConfigUtils
+import nl.lumc.sasc.biopet.utils.config.Config
+import org.broadinstitute.gatk.queue.QSettings
+import org.scalatest.Matchers
+import org.scalatest.testng.TestNGSuite
+import org.testng.annotations.Test
+
+/**
+ * Created by pjvan_thof on 13-5-16.
+ */
+class GenerateIndexesTest extends TestNGSuite with Matchers {
+  def initPipeline(map: Map[String, Any]): GenerateIndexes = {
+    new GenerateIndexes() {
+      override def configNamespace = "generateindexes"
+      override def globalConfig = new Config(ConfigUtils.mergeMaps(map, GenerateIndexesTest.config))
+      qSettings = new QSettings
+      qSettings.runName = "test"
+    }
+  }
+
+  @Test
+  def testNoFastaUri: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("" -> "")))
+    intercept[IllegalArgumentException] {
+      pipeline.script()
+    }
+  }
+
+  @Test
+  def testSingleFasta: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("fasta_uri" -> "uri")))
+    pipeline.script()
+  }
+
+  @Test
+  def testMultiFasta: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("fasta_uri" -> List("uri", "uri2", "uri3.gz"))))
+    pipeline.script()
+  }
+
+  @Test
+  def testSingleDbsnp: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("fasta_uri" -> "uri", "dbsnp_vcf_uri" -> "uri.vcf.gz")))
+    pipeline.script()
+  }
+
+  @Test
+  def testMultiDbsnp: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("fasta_uri" -> "uri", "dbsnp_vcf_uri" -> List("uri.vcf.gz", "uri2.vcf.gz"))))
+    pipeline.script()
+  }
+
+  @Test
+  def testVep: Unit = {
+    val pipeline = initPipeline(Map())
+    pipeline.referenceConfig = Map("s1" -> Map("g1" -> Map("fasta_uri" -> "uri", "vep_cache_uri" -> "something/human_vep_80_hg19.tar.gz")))
+    pipeline.script()
+  }
+
+}
+
+object GenerateIndexesTest {
+  val outputDir = Files.createTempDir()
+  outputDir.deleteOnExit()
+
+  val config = Map("output_dir" -> outputDir,
+    "bwa" -> Map("exe" -> "test"),
+    "star" -> Map("exe" -> "test"),
+    "bowtiebuild" -> Map("exe" -> "test"),
+    "bowtie2build" -> Map("exe" -> "test"),
+    "gmapbuild" -> Map("exe" -> "test"),
+    "samtools" -> Map("exe" -> "test"),
+    "md5sum" -> Map("exe" -> "test"),
+    "gatk_jar" -> "test",
+    "tabix" -> Map("exe" -> "test")
+  )
+}

shiva/src/test/scala/nl/lumc/sasc/biopet/pipelines/shiva/ShivaTest.scala

@@ -161,6 +161,7 @@ class ShivaWithAnnotationTest extends ShivaTestTrait {
 
 object ShivaTest {
   val outputDir = Files.createTempDir()
+  outputDir.deleteOnExit()
   new File(outputDir, "input").mkdirs()
   def inputTouch(name: String): String = {
     val file = new File(outputDir, "input" + File.separator + name)