Commit 2150ef84 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Merge branch 'develop' into feature-basty_generate_fasta

Conflicts:
	biopet-framework/src/main/scala/nl/lumc/sasc/biopet/core/BiopetExecutable.scala
parents db4ba62f 4805a430
<component name="libraryTable">
<library name="Maven: org.scala-lang:scala-library:2.11.2">
<library name="Maven: org.scala-lang:scala-library:2.11.2" type="Scala">
<properties>
<compiler-classpath>
<root url="file://$MAVEN_REPOSITORY$/org/scala-lang/scala-compiler/2.11.2/scala-compiler-2.11.2.jar" />
<root url="file://$MAVEN_REPOSITORY$/org/scala-lang/scala-library/2.11.2/scala-library-2.11.2.jar" />
<root url="file://$MAVEN_REPOSITORY$/org/scala-lang/scala-reflect/2.11.2/scala-reflect-2.11.2.jar" />
</compiler-classpath>
</properties>
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/org/scala-lang/scala-library/2.11.2/scala-library-2.11.2.jar!/" />
</CLASSES>
......
......@@ -5,6 +5,7 @@
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/org/scalaz/scalaz-core_2.11/7.1.0/scalaz-core_2.11-7.1.0-javadoc.jar!/" />
<root url="http://docs.typelevel.org/api/scalaz/stable/7.1.0-M3/doc/#package" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/org/scalaz/scalaz-core_2.11/7.1.0/scalaz-core_2.11-7.1.0-sources.jar!/" />
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ScalaCompilerConfiguration">
<option name="deprecationWarnings" value="true" />
<parameters>
<parameter value="-dependencyfile" />
<parameter value="$MODULE_DIR$/target/.scala_dependencies" />
<parameter value="-feature" />
</parameters>
</component>
</project>
\ No newline at end of file
......@@ -54,29 +54,43 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
}
protected def checkExecutable {
try if (executable != null) {
val buffer = new StringBuffer()
val cmd = Seq("which", executable)
val process = Process(cmd).run(ProcessLogger(buffer.append(_)))
if (process.exitValue == 0) {
executable = buffer.toString
val file = new File(executable)
executable = file.getCanonicalPath
} else {
logger.error("executable: '" + executable + "' not found, please check config")
throw new QException("executable: '" + executable + "' not found, please check config")
if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
try if (executable != null) {
if (!BiopetCommandLineFunctionTrait.executableCache.contains(executable)) {
val oldExecutable = executable
val buffer = new StringBuffer()
val cmd = Seq("which", executable)
val process = Process(cmd).run(ProcessLogger(buffer.append(_)))
if (process.exitValue == 0) {
executable = buffer.toString
val file = new File(executable)
executable = file.getCanonicalPath
} else {
logger.error("executable: '" + executable + "' not found, please check config")
throw new QException("executable: '" + executable + "' not found, please check config")
}
BiopetCommandLineFunctionTrait.executableCache += oldExecutable -> executable
BiopetCommandLineFunctionTrait.executableCache += executable -> executable
} else {
executable = BiopetCommandLineFunctionTrait.executableCache(executable)
}
if (!BiopetCommandLineFunctionTrait.executableMd5Cache.contains(executable)) {
val is = new FileInputStream(executable)
val cnt = is.available
val bytes = Array.ofDim[Byte](cnt)
is.read(bytes)
is.close()
val temp = MessageDigest.getInstance("MD5").digest(bytes).map("%02X".format(_)).mkString.toLowerCase
BiopetCommandLineFunctionTrait.executableMd5Cache += executable -> temp
}
addJobReportBinding("md5sum_exe", BiopetCommandLineFunctionTrait.executableMd5Cache(executable))
} catch {
case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
}
val is = new FileInputStream(executable)
val cnt = is.available
val bytes = Array.ofDim[Byte](cnt)
is.read(bytes)
is.close()
val md5: String = MessageDigest.getInstance("MD5").digest(bytes).map("%02X".format(_)).mkString.toLowerCase
addJobReportBinding("md5sum_exe", md5)
} catch {
case ioe: java.io.IOException => logger.warn("Could not use 'which', check on executable skipped: " + ioe)
} else {
addJobReportBinding("md5sum_exe", BiopetCommandLineFunctionTrait.executableMd5Cache(executable))
}
}
......@@ -138,4 +152,6 @@ trait BiopetCommandLineFunctionTrait extends CommandLineFunction with Configurab
object BiopetCommandLineFunctionTrait {
import scala.collection.mutable.Map
private val versionCache: Map[String, String] = Map()
private val executableMd5Cache: Map[String, String] = Map()
private val executableCache: Map[String, String] = Map()
}
\ No newline at end of file
package nl.lumc.sasc.biopet.core
import java.io.File
import java.util.Properties
import nl.lumc.sasc.biopet.core.config.Config
import org.apache.log4j.Logger
object BiopetExecutable extends Logging {
......@@ -35,7 +37,9 @@ object BiopetExecutable extends Logging {
nl.lumc.sasc.biopet.tools.SageCountFastq,
nl.lumc.sasc.biopet.tools.SageCreateLibrary,
nl.lumc.sasc.biopet.tools.SageCreateTagCounts,
nl.lumc.sasc.biopet.tools.BastyGenerateFasta)
nl.lumc.sasc.biopet.tools.BastyGenerateFasta,
nl.lumc.sasc.biopet.tools.MergeAlleles,
nl.lumc.sasc.biopet.tools.SamplesTsvToJson)
)
/**
......@@ -88,6 +92,14 @@ object BiopetExecutable extends Logging {
println("version: " + getVersion)
}
case Array(module, name, passArgs @ _*) => {
// Reading config files
val argsSize = passArgs.size
for (t <- 0 until argsSize) {
if (passArgs(t) == "-config" || args(t) == "--config_file") {
if (t >= argsSize) throw new IllegalStateException("-config needs a value")
Config.global.loadConfigFile(new File(passArgs(t + 1)))
}
}
getCommand(module, name).main(passArgs.toArray)
}
case Array(module) => {
......
......@@ -5,6 +5,10 @@ import org.broadinstitute.gatk.queue.function.JavaCommandLineFunction
trait BiopetJavaCommandLineFunction extends JavaCommandLineFunction with BiopetCommandLineFunctionTrait {
executable = "java"
javaGCThreads = config("java_gc_threads")
javaGCHeapFreeLimit = config("java_gc_heap_freelimit")
javaGCTimeLimit = config("java_gc_timelimit")
override def afterGraph {
memoryLimit = config("memory_limit")
}
......
......@@ -2,7 +2,7 @@ package nl.lumc.sasc.biopet.core
import java.io.File
import java.io.PrintWriter
import nl.lumc.sasc.biopet.core.config.Configurable
import nl.lumc.sasc.biopet.core.config.{ Config, Configurable }
import org.broadinstitute.gatk.utils.commandline.Argument
import org.broadinstitute.gatk.queue.QSettings
import org.broadinstitute.gatk.queue.function.QFunction
......@@ -25,7 +25,6 @@ trait BiopetQScript extends Configurable {
var functions: Seq[QFunction]
final def script() {
for (file <- configfiles) globalConfig.loadConfigFile(file)
if (!outputDir.endsWith("/")) outputDir += "/"
init
biopetScript
......@@ -33,7 +32,7 @@ trait BiopetQScript extends Configurable {
case f: BiopetCommandLineFunctionTrait => f.afterGraph
case _ =>
}
val configReport = globalConfig.getReport
val configReport = Config.global.getReport
val configReportFile = new File(outputDir + qSettings.runName + ".configreport.txt")
configReportFile.getParentFile.mkdir
val writer = new PrintWriter(configReportFile)
......
......@@ -7,9 +7,13 @@ import org.apache.log4j.WriterAppender
import org.apache.log4j.helpers.DateLayout
trait Logging {
protected val logger = Logger.getLogger(getClass.getSimpleName.split("\\$").last)
def logger = Logging.logger
}
object Logging {
val logger = Logger.getLogger("Logging")
private[core] val logLayout = new DateLayout() {
val logLayout = new DateLayout() {
val ignoresThrowable = false
def format(event: org.apache.log4j.spi.LoggingEvent): String = {
val calendar: Calendar = Calendar.getInstance
......@@ -18,10 +22,13 @@ trait Logging {
val formattedDate: String = formatter.format(calendar.getTime)
var logLevel = event.getLevel.toString
while (logLevel.size < 6) logLevel += " "
logLevel + " [" + formattedDate + "] [" + event.getLoggerName + "] " + event.getMessage + "\n"
val className = event.getLocationInformation.getClassName.split("\\.").last.split("\\$").head
logLevel + " [" + formattedDate + "] [" + className + "] " + event.getMessage + "\n"
}
}
private[core] val stderrAppender = new WriterAppender(logLayout, sys.process.stderr)
val stderrAppender = new WriterAppender(logLayout, sys.process.stderr)
logger.setLevel(org.apache.log4j.Level.INFO)
logger.addAppender(stderrAppender)
}
logger.addAppender(Logging.stderrAppender)
}
\ No newline at end of file
package nl.lumc.sasc.biopet.core
import nl.lumc.sasc.biopet.core.config.Configurable
import nl.lumc.sasc.biopet.core.config.{ Config, Configurable }
trait MultiSampleQScript extends BiopetQScript {
type LibraryOutput <: AbstractLibraryOutput
......@@ -20,7 +20,7 @@ trait MultiSampleQScript extends BiopetQScript {
final def runSamplesJobs() {
samplesConfig = config("samples")
if (samplesConfig == null) samplesConfig = Map()
if (globalConfig.contains("samples")) for ((key, value) <- samplesConfig) {
if (Config.global.contains("samples")) for ((key, value) <- samplesConfig) {
var sample = Configurable.any2map(value)
if (!sample.contains("ID")) sample += ("ID" -> key)
if (sample("ID") == key) {
......
package nl.lumc.sasc.biopet.core
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
import nl.lumc.sasc.biopet.core.workaround.BiopetQCommandLine
trait PipelineCommand extends MainCommand with GatkLogging {
......
......@@ -9,10 +9,10 @@ trait ToolCommand extends MainCommand with Logging {
abstract class AbstractOptParser extends scopt.OptionParser[Args](commandName) {
opt[Unit]("log_nostderr") foreach { _ =>
logger.removeAppender(stderrAppender)
logger.removeAppender(Logging.stderrAppender)
} text ("No output to stderr")
opt[File]("log_file") foreach { x =>
logger.addAppender(new WriterAppender(logLayout, new java.io.PrintStream(x)))
logger.addAppender(new WriterAppender(Logging.logLayout, new java.io.PrintStream(x)))
} text ("Log file") valueName ("<file>")
opt[String]('l', "log_level") foreach { x =>
x.toLowerCase match {
......
package nl.lumc.sasc.biopet.core.config
import java.io.File
import org.broadinstitute.gatk.queue.util.Logging
import nl.lumc.sasc.biopet.core.Logging
import argonaut._, Argonaut._
import scalaz._, Scalaz._
......@@ -104,7 +104,9 @@ class Config(var map: Map[String, Any]) extends Logging {
override def toString(): String = map.toString
}
object Config {
object Config extends Logging {
val global = new Config
def valueToMap(input: Any): Map[String, Any] = {
input match {
case m: Map[_, _] => return m.asInstanceOf[Map[String, Any]]
......@@ -197,4 +199,28 @@ object Config {
return None
}
}
def mapToJson(map: Map[String, Any]): Json = {
map.foldLeft(jEmptyObject)((acc, kv) => (kv._1 := {
kv._2 match {
case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
case _ => anyToJson(kv._2)
}
}) ->: acc)
}
def anyToJson(any: Any): Json = {
any match {
case j: Json => j
case m: Map[_, _] => mapToJson(m.map(m => m._1.toString -> anyToJson(m._2)))
case l: List[_] => Json.array(l.map(anyToJson(_)): _*)
case n: Int => Json.jNumberOrString(n)
case n: Double => Json.jNumberOrString(n)
case n: Long => Json.jNumberOrString(n)
case n: Short => Json.jNumberOrString(n)
case n: Float => Json.jNumberOrString(n)
case n: Byte => Json.jNumberOrString(n)
case _ => jString(any.toString)
}
}
}
\ No newline at end of file
......@@ -6,16 +6,16 @@ import scala.language.implicitConversions
trait Configurable extends Logging {
val root: Configurable
val globalConfig: Config = if (root != null) root.globalConfig else new Config()
//val globalConfig: Config = if (root != null) root.globalConfig else new Config()
def configPath: List[String] = if (root != null) root.configFullPath else List()
protected lazy val configName = getClass.getSimpleName.toLowerCase
protected lazy val configFullPath = configName :: configPath
var defaults: scala.collection.mutable.Map[String, Any] = if (root != null) scala.collection.mutable.Map(root.defaults.toArray: _*)
else scala.collection.mutable.Map()
val config = new ConfigFuntions
val config = new ConfigFunctions
protected class ConfigFuntions {
protected class ConfigFunctions {
def apply(key: String, default: Any = null, submodule: String = null, required: Boolean = false, freeVar: Boolean = true): ConfigValue = {
val m = if (submodule != null) submodule else configName
val p = if (submodule != null) configName :: configPath else configPath
......@@ -29,15 +29,15 @@ trait Configurable extends Logging {
throw new IllegalStateException("Value in config could not be found but it is required, key: " + key + " module: " + m + " path: " + p)
} else return null
}
if (d == null) return globalConfig(m, p, key, freeVar)
else return globalConfig(m, p, key, d, freeVar)
if (d == null) return Config.global(m, p, key, freeVar)
else return Config.global(m, p, key, d, freeVar)
}
def contains(key: String, submodule: String = null, freeVar: Boolean = true) = {
val m = if (submodule != null) submodule else configName
val p = if (submodule != null) configName :: configPath else configPath
globalConfig.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
Config.global.contains(m, p, key, freeVar) || !(Config.getValueFromMap(defaults.toMap, ConfigValueIndex(m, p, key, freeVar)) == None)
}
}
......
......@@ -24,7 +24,7 @@
* THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package nl.lumc.sasc.biopet.core
package nl.lumc.sasc.biopet.core.workaround
import java.io.File
import org.broadinstitute.gatk.utils.commandline._
......@@ -32,6 +32,7 @@ import org.broadinstitute.gatk.queue.util._
import org.broadinstitute.gatk.queue.QCommandPlugin
import org.broadinstitute.gatk.queue.QScript
import org.broadinstitute.gatk.queue.QScriptManager
import org.broadinstitute.gatk.queue.util.{ Logging => GatkLogging }
import org.broadinstitute.gatk.queue.engine.{ QStatusMessenger, QGraphSettings, QGraph }
import collection.JavaConversions._
import org.broadinstitute.gatk.utils.classloader.PluginManager
......@@ -43,11 +44,12 @@ import java.net.URL
import java.util.{ ResourceBundle, Arrays }
import org.broadinstitute.gatk.utils.text.TextFormattingUtils
import org.apache.commons.io.FilenameUtils
import nl.lumc.sasc.biopet.core.BiopetExecutable
/**
* Entry point of Queue. Compiles and runs QScripts passed in to the command line.
*/
object BiopetQCommandLine extends Logging {
object BiopetQCommandLine extends GatkLogging {
/**
* Main.
* @param argv Arguments.
......
......@@ -11,7 +11,7 @@ class HaplotypeCaller(val root: Configurable) extends org.broadinstitute.gatk.qu
if (config.contains("scattercount")) scatterCount = config("scattercount")
if (config.contains("dbsnp")) this.dbsnp = config("dbsnp")
this.sample_ploidy = config("ploidy")
nct = config("threads", default = 3)
nct = config("threads", default = 1)
bamOutput = config("bamOutput")
memoryLimit = Option(nct.getOrElse(1) * 2)
if (config.contains("allSitePLs")) this.allSitePLs = config("allSitePLs")
......
......@@ -10,7 +10,7 @@ class UnifiedGenotyper(val root: Configurable) extends org.broadinstitute.gatk.q
if (config.contains("scattercount")) scatterCount = config("scattercount")
if (config.contains("dbsnp")) this.dbsnp = config("dbsnp")
this.sample_ploidy = config("ploidy")
nct = config("threads", default = 3)
nct = config("threads", default = 1)
memoryLimit = Option(nct.getOrElse(1) * 2)
if (config.contains("allSitePLs")) this.allSitePLs = config("allSitePLs")
if (config.contains("output_mode")) {
......
......@@ -26,7 +26,6 @@ class BamMetrics(val root: Configurable) extends QScript with BiopetQScript {
var wholeGenome = false
def init() {
for (file <- configfiles) globalConfig.loadConfigFile(file)
if (outputDir == null) throw new IllegalStateException("Missing Output directory on BamMetrics module")
else if (!outputDir.endsWith("/")) outputDir += "/"
if (config.contains("target_bed")) {
......
......@@ -18,10 +18,10 @@ class Flexiprep(val root: Configurable) extends QScript with BiopetQScript {
var input_R2: File = _
@Argument(doc = "Skip Trim fastq files", shortName = "skiptrim", required = false)
var skipTrim: Boolean = false
var skipTrim: Boolean = config("skiptrim", default = false)
@Argument(doc = "Skip Clip fastq files", shortName = "skipclip", required = false)
var skipClip: Boolean = false
var skipClip: Boolean = config("skipclip", default = false)
@Argument(doc = "Sample name", shortName = "sample", required = true)
var sampleName: String = _
......@@ -43,9 +43,6 @@ class Flexiprep(val root: Configurable) extends QScript with BiopetQScript {
val summary = new FlexiprepSummary(this)
def init() {
for (file <- configfiles) globalConfig.loadConfigFile(file)
if (!skipTrim) skipTrim = config("skiptrim", default = false)
if (!skipClip) skipClip = config("skipclip", default = false)
if (input_R1 == null) throw new IllegalStateException("Missing R1 on flexiprep module")
if (outputDir == null) throw new IllegalStateException("Missing Output directory on flexiprep module")
if (sampleName == null) throw new IllegalStateException("Missing Sample name on flexiprep module")
......
......@@ -18,18 +18,15 @@ class GatkBenchmarkGenotyping(val root: Configurable) extends QScript with Biope
@Input(doc = "Gvcf files", shortName = "I", required = false)
var gvcfFiles: List[File] = Nil
@Argument(doc = "Reference", shortName = "R", required = false)
var reference: File = _
var reference: File = config("reference")
@Argument(doc = "Dbsnp", shortName = "dbsnp", required = false)
var dbsnp: File = _
var dbsnp: File = config("dbsnp")
def init() {
if (config.contains("gvcffiles")) for (file <- config("gvcffiles").getList) {
gvcfFiles ::= file.toString
}
if (reference == null) reference = config("reference")
if (dbsnp == null) dbsnp = config("dbsnp")
if (outputDir == null) throw new IllegalStateException("Missing Output directory on gatk module")
else if (!outputDir.endsWith("/")) outputDir += "/"
}
......
......@@ -13,10 +13,10 @@ class GatkGenotyping(val root: Configurable) extends QScript with BiopetQScript
var inputGvcfs: List[File] = Nil
@Argument(doc = "Reference", shortName = "R", required = false)
var reference: File = _
var reference: File = config("reference")
@Argument(doc = "Dbsnp", shortName = "dbsnp", required = false)
var dbsnp: File = _
var dbsnp: File = config("dbsnp")
@Argument(doc = "OutputName", required = false)
var outputName: String = "genotype"
......@@ -28,8 +28,6 @@ class GatkGenotyping(val root: Configurable) extends QScript with BiopetQScript
var samples: List[String] = Nil
def init() {
if (reference == null) reference = config("reference")
if (dbsnp == null) dbsnp = config("dbsnp")
if (outputFile == null) outputFile = outputDir + outputName + ".vcf.gz"
if (outputDir == null) throw new IllegalStateException("Missing Output directory on gatk module")
else if (!outputDir.endsWith("/")) outputDir += "/"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment