Merge branch 'develop' into fix-BIOPET-582

Jenkinsfile

@@ -17,10 +17,14 @@ node('local') {
             }
         }
 
-        stage('Report tests') {
+        stage('Report Tests') {
             junit '*/target/surefire-reports/*.xml'
         }
 
+        stage('Check Documentation') {
+            sh 'mkdocs build --clean --strict'
+        }
+
         if(currentBuild.result == null || "SUCCESS".equals(currentBuild.result)) {
             currentBuild.result = "SUCCESS"
             slackSend (color: '#00FF00', message: "${currentBuild.result}: Job '${env.JOB_NAME} #${env.BUILD_NUMBER}' (<${env.BUILD_URL}|Open>)", channel: '#biopet-bot', teamDomain: 'lumc', tokenCredentialId: 'lumc')

README.md

@@ -14,7 +14,7 @@ Full documantation is here: [Biopet documantation](http://biopet-docs.readthedoc
 Biopet is available as a JAR package in SHARK. The easiest way to start using it is to activate the `biopet` environment module, which sets useful aliases and environment variables:
 
 ~~~
-$ module load biopet/v0.6.0
+$ module load biopet/v0.8.0
 ~~~
 
 With each Biopet release, an accompanying environment module is also released. The latest release is version 0.4.0, thus `biopet/v0.4.0` is the module you would want to load.

bam2wig/pom.xml

@@ -26,7 +26,7 @@
     <parent>
         <groupId>nl.lumc.sasc</groupId>
         <artifactId>Biopet</artifactId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
         <relativePath>../</relativePath>
     </parent>
 

bammetrics/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>nl.lumc.sasc</groupId>
         <artifactId>Biopet</artifactId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
         <relativePath>../</relativePath>
     </parent>
     

basty/pom.xml

@@ -31,7 +31,7 @@
     <parent>
         <groupId>nl.lumc.sasc</groupId>
         <artifactId>Biopet</artifactId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
         <relativePath>../</relativePath>
     </parent>
     

basty/src/test/scala/nl/lumc/sasc/biopet/pipelines/basty/BastyTest.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.pipelines.basty
 
 import java.io.{ File, FileOutputStream }

biopet-core/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>Biopet</artifactId>
         <groupId>nl.lumc.sasc</groupId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
         <relativePath>../</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/BiopetQScript.scala

@@ -93,15 +93,21 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
     }
 
     logger.info("Running pre commands")
-    for (function <- functions) function match {
-      case f: BiopetCommandLineFunction =>
-        f.preProcessExecutable()
-        f.beforeGraph()
-        f.internalBeforeGraph()
-        f.commandLine
-      case f: WriteSummary => f.init()
-      case _               =>
+    var count = 0
+    for (function <- functions) {
+      function match {
+        case f: BiopetCommandLineFunction =>
+          f.preProcessExecutable()
+          f.beforeGraph()
+          f.internalBeforeGraph()
+          f.commandLine
+        case f: WriteSummary => f.init()
+        case _               =>
+      }
+      count += 1
+      if (count % 500 == 0) logger.info(s"Preprocessing done for ${count} jobs out of ${functions.length} total")
     }
+    logger.info(s"Preprocessing done for ${functions.length} functions")
 
     val logDir = new File(outputDir, ".log" + File.separator + qSettings.runName.toLowerCase)
 
@@ -116,6 +122,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
       if (!i.file.isAbsolute) Logging.addError(s"Input file should be an absolute path: ${i.file}")
     }
 
+    logger.info("Set stdout file when not set")
     functions.filter(_.jobOutputFile == null).foreach(f => {
       val className = if (f.getClass.isAnonymousClass) f.getClass.getSuperclass.getSimpleName else f.getClass.getSimpleName
       BiopetQScript.safeOutputs(f) match {
@@ -127,6 +134,7 @@ trait BiopetQScript extends Configurable with GatkLogging { qscript: QScript =>
     if (!skipWriteDependencies) WriteDependencies.writeDependencies(
       functions,
       new File(logDir, "graph"))
+    else logger.debug("Write dependencies is skipped")
 
     Logging.checkErrors()
     logger.info("Script complete without errors")

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/PipelineStatus.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.core
 
 import java.io.{ File, PrintWriter }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/TemplateTool.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.core
 
 import java.io.{ File, PrintWriter }

biopet-core/src/main/scala/nl/lumc/sasc/biopet/core/WriteDependencies.scala

@@ -116,11 +116,11 @@ object WriteDependencies extends Logging with Configurable {
           case s: WriteSummary if s.qscript.root == null => true
           case _                                         => false
         }), "intermediate" -> f.isIntermediate,
-          "depends_on_intermediate" -> BiopetQScript.safeOutputs(f).getOrElse(Seq()).exists(files(_).isIntermediate),
-          "depends_on_jobs" -> BiopetQScript.safeOutputs(f).getOrElse(Seq()).toList.flatMap(files(_).outputJobNames).distinct,
+          "depends_on_intermediate" -> BiopetQScript.safeInputs(f).getOrElse(Seq()).exists(files(_).isIntermediate),
+          "depends_on_jobs" -> BiopetQScript.safeInputs(f).getOrElse(Seq()).toList.flatMap(files(_).outputJobNames).distinct,
           "output_used_by_jobs" -> BiopetQScript.safeOutputs(f).getOrElse(Seq()).toList.flatMap(files(_).inputJobNames).distinct,
           "outputs" -> (f.jobOutputFile :: BiopetQScript.safeOutputs(f).getOrElse(Seq()).toList),
-          "inputs" -> BiopetQScript.safeOutputs(f).getOrElse(Seq()).toList,
+          "inputs" -> BiopetQScript.safeInputs(f).getOrElse(Seq()).toList,
           "done_files" -> BiopetQScript.safeDoneFiles(f).getOrElse(Seq()).toList,
           "fail_files" -> BiopetQScript.safeFailFiles(f).getOrElse(Seq()).toList,
           "stdout_file" -> f.jobOutputFile,

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/PipelineStatusTest.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.core
 
 import java.io.{ File, PrintWriter }

biopet-core/src/test/scala/nl/lumc/sasc/biopet/core/annotation/AnnotationTest.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.core.annotation
 
 import java.io.File

biopet-extensions/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>Biopet</artifactId>
         <groupId>nl.lumc.sasc</groupId>
-        <version>0.8.0-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

biopet-extensions/src/main/resources/nl/lumc/sasc/biopet/extensions/freec/freec_CNVPlot.py

+#
+# Biopet is built on top of GATK Queue for building bioinformatic
+# pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+# SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+# should also be able to execute Biopet tools and pipelines.
+#
+# Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+#
+# Contact us at: sasc@lumc.nl
+#
+# A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+# license; For commercial users or users who do not want to follow the AGPL
+# license, please contact us to obtain a separate license.
+#
+
 import argparse
 from os.path import join
 

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/GffRead.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.extensions
 
 import java.io.File

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/Sed.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.extensions
 
 import java.io.File

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/centrifuge/Centrifuge.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.extensions.centrifuge
 
 import java.io.File

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/centrifuge/CentrifugeKreport.scala

+/**
+ * Biopet is built on top of GATK Queue for building bioinformatic
+ * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
+ * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
+ * should also be able to execute Biopet tools and pipelines.
+ *
+ * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
+ *
+ * Contact us at: sasc@lumc.nl
+ *
+ * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
+ * license; For commercial users or users who do not want to follow the AGPL
+ * license, please contact us to obtain a separate license.
+ */
 package nl.lumc.sasc.biopet.extensions.centrifuge
 
 import java.io.File

biopet-extensions/src/main/scala/nl/lumc/sasc/biopet/extensions/delly/DellyCaller.scala

@@ -41,11 +41,6 @@ class DellyCaller(val root: Configurable) extends BiopetCommandLineFunction with
   @Argument(doc = "What kind of analysis to run: DEL,DUP,INV,TRA")
   var analysistype: String = _
 
-  override def beforeGraph(): Unit = {
-    super.beforeGraph()
-    deps ::= new File(input.getParentFile, input.getName + ".bai")
-  }
-
   def cmdLine = required(executable) +
     required("-t", analysistype) +
     required("-o", outputvcf) +