Commit 385c71e9 authored by Peter van 't Hof's avatar Peter van 't Hof
Browse files

Biopet core code inspection round

parent c181bccd
...@@ -44,7 +44,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript => ...@@ -44,7 +44,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
private[summary] var summaryQScripts: List[SummaryQScript] = Nil private[summary] var summaryQScripts: List[SummaryQScript] = Nil
/** Name of the pipeline in the summary */ /** Name of the pipeline in the summary */
var summaryName = configNamespace var summaryName: String = configNamespace
/** Must return a map with used settings for this pipeline */ /** Must return a map with used settings for this pipeline */
def summarySettings: Map[String, Any] def summarySettings: Map[String, Any]
...@@ -98,7 +98,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript => ...@@ -98,7 +98,7 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
Await.result( Await.result(
SummaryDb.openSqliteSummary(summaryDbFile).getRuns(runId = Some(id)).map(_.headOption), SummaryDb.openSqliteSummary(summaryDbFile).getRuns(runId = Some(id)).map(_.headOption),
Duration.Inf) match { Duration.Inf) match {
case Some(x) => id case Some(_) => id
case _ => case _ =>
logger.warn( logger.warn(
s"Run id found in '$runIdFile' does not exist in summary, creating a new run") s"Run id found in '$runIdFile' does not exist in summary, creating a new run")
...@@ -191,22 +191,22 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript => ...@@ -191,22 +191,22 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
qscript match { qscript match {
case q: MultiSampleQScript => case q: MultiSampleQScript =>
// Global level // Global level
for ((key, file) <- qscript.summaryFiles) addChecksum(file) for ((_, file) <- qscript.summaryFiles) addChecksum(file)
for ((sampleName, sample) <- q.samples) { for ((_, sample) <- q.samples) {
// Sample level // Sample level
for ((key, file) <- sample.summaryFiles) addChecksum(file) for ((_, file) <- sample.summaryFiles) addChecksum(file)
for ((libName, lib) <- sample.libraries) { for ((_, lib) <- sample.libraries) {
// Library level // Library level
for ((key, file) <- lib.summaryFiles) addChecksum(file) for ((_, file) <- lib.summaryFiles) addChecksum(file)
} }
} }
case q => for ((key, file) <- q.summaryFiles) addChecksum(file) case q => for ((_, file) <- q.summaryFiles) addChecksum(file)
} }
for (inputFile <- inputFiles) { for (inputFile <- inputFiles) {
inputFile.md5 match { inputFile.md5 match {
case Some(checksum) => { case Some(checksum) =>
val checkMd5 = new CheckChecksum val checkMd5 = new CheckChecksum
checkMd5.inputFile = inputFile.file checkMd5.inputFile = inputFile.file
if (!SummaryQScript.md5sumCache.contains(inputFile.file)) if (!SummaryQScript.md5sumCache.contains(inputFile.file))
...@@ -216,7 +216,6 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript => ...@@ -216,7 +216,6 @@ trait SummaryQScript extends BiopetQScript { qscript: QScript =>
checkMd5.jobOutputFile = new File(checkMd5.checksumFile.getParentFile, checkMd5.jobOutputFile = new File(checkMd5.checksumFile.getParentFile,
checkMd5.checksumFile.getName + ".check.out") checkMd5.checksumFile.getName + ".check.out")
add(checkMd5) add(checkMd5)
}
case _ => case _ =>
} }
} }
......
...@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.Input ...@@ -25,7 +25,7 @@ import org.broadinstitute.gatk.utils.commandline.Input
import scala.collection.mutable import scala.collection.mutable
import scala.io.Source import scala.io.Source
import scala.concurrent.Await import scala.concurrent.{Await, Future}
import scala.concurrent.duration.Duration import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.ExecutionContext.Implicits.global
...@@ -40,7 +40,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co ...@@ -40,7 +40,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
require(parent != null) require(parent != null)
/** To access qscript for this summary */ /** To access qscript for this summary */
val qscript = parent val qscript: SummaryQScript = parent
@Input(doc = "deps", required = false) @Input(doc = "deps", required = false)
var deps: List[File] = Nil var deps: List[File] = Nil
...@@ -62,28 +62,27 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co ...@@ -62,28 +62,27 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
val db = SummaryDb.openSqliteSummary(qscript.summaryDbFile) val db = SummaryDb.openSqliteSummary(qscript.summaryDbFile)
if (qscript == root) { // This initialize the database if (qscript == root) { // This initialize the database
qscript match { qscript match {
case s: MultiSampleQScript => s.initSummaryDb case s: MultiSampleQScript => s.initSummaryDb()
case t: SampleLibraryTag => case t: SampleLibraryTag =>
t.sampleId.foreach { t.sampleId.foreach {
case sampleName => sampleName =>
val sampleId = Await val sampleId = Await
.result(db.getSamples(name = Some(sampleName), runId = Some(qscript.summaryRunId)) .result(db.getSamples(name = Some(sampleName), runId = Some(qscript.summaryRunId))
.map(_.headOption.map(_.id)), .map(_.headOption.map(_.id)),
Duration.Inf) Duration.Inf)
.getOrElse { .getOrElse {
Await.result(db.createOrUpdateSample(sampleName, qscript.summaryRunId), Await.result(db.createOrUpdateSample(sampleName, qscript.summaryRunId),
Duration.Inf) Duration.Inf)
} }
t.libId.foreach { libName => t.libId.foreach { libName =>
val libId = Await Await.result(db.getSamples(name = Some(libName),
.result(db.getSamples(name = Some(libName), runId = Some(qscript.summaryRunId),
runId = Some(qscript.summaryRunId), sampleId = Some(sampleId))
sampleId = Some(sampleId)) .map(_.headOption.map(_.id)),
.map(_.headOption.map(_.id)), Duration.Inf)
Duration.Inf)
.getOrElse { .getOrElse {
Await.result(db.createOrUpdateLibrary(libName, qscript.summaryRunId, sampleId), Await.result(db.createOrUpdateLibrary(libName, qscript.summaryRunId, sampleId),
Duration.Inf) Duration.Inf)
} }
} }
} }
...@@ -102,7 +101,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co ...@@ -102,7 +101,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
try { try {
deps :+= f.firstOutput deps :+= f.firstOutput
} catch { } catch {
case e: NullPointerException => logger.debug("Queue values are not initialized") case _: NullPointerException => logger.debug("Queue values are not initialized")
} }
case _ => case _ =>
} }
...@@ -328,7 +327,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co ...@@ -328,7 +327,7 @@ class WriteSummary(val parent: SummaryQScript) extends InProcessFunction with Co
} }
/** Convert summarizable to a summary map */ /** Convert summarizable to a summary map */
def parseSummarizable(summarizable: Summarizable, name: String) = { def parseSummarizable(summarizable: Summarizable, name: String): Map[String, Any] = {
val stats = summarizable.summaryStats val stats = summarizable.summaryStats
val files = parseFiles(summarizable.summaryFiles) val files = parseFiles(summarizable.summaryFiles)
...@@ -365,11 +364,11 @@ object WriteSummary { ...@@ -365,11 +364,11 @@ object WriteSummary {
libId: Option[Int], libId: Option[Int],
key: String, key: String,
file: File, file: File,
outputDir: File) = { outputDir: File): Future[Int] = {
val path = if (file.getAbsolutePath.startsWith(outputDir.getAbsolutePath + File.separator)) { val path = if (file.getAbsolutePath.startsWith(outputDir.getAbsolutePath + File.separator)) {
"." + file.getAbsolutePath.stripPrefix(s"${outputDir.getAbsolutePath}") "." + file.getAbsolutePath.stripPrefix(s"${outputDir.getAbsolutePath}")
} else file.getAbsolutePath } else file.getAbsolutePath
val md5 = SummaryQScript.md5sumCache.get(file).map(WriteSummary.parseChecksum(_)) val md5 = SummaryQScript.md5sumCache.get(file).map(WriteSummary.parseChecksum)
val size = if (file.exists()) file.length() else 0L val size = if (file.exists()) file.length() else 0L
val link = if (file.exists()) java.nio.file.Files.isSymbolicLink(file.toPath) else false val link = if (file.exists()) java.nio.file.Files.isSymbolicLink(file.toPath) else false
db.createOrUpdateFile(runId, db.createOrUpdateFile(runId,
......
...@@ -98,7 +98,7 @@ object BiopetQCommandLine extends GatkLogging { ...@@ -98,7 +98,7 @@ object BiopetQCommandLine extends GatkLogging {
System.exit(CommandLineProgram.result) System.exit(CommandLineProgram.result)
} }
val timestamp = System.currentTimeMillis val timestamp: Long = System.currentTimeMillis
} }
/** /**
...@@ -135,7 +135,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -135,7 +135,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
// override createByType to pass the correct exceptions // override createByType to pass the correct exceptions
new PluginManager[QScript](qPluginType, List(qScriptClasses.toURI.toURL)) { new PluginManager[QScript](qPluginType, List(qScriptClasses.toURI.toURL)) {
override def createByType(plugintype: Class[_ <: QScript]) = { override def createByType(plugintype: Class[_ <: QScript]): QScript = {
val noArgsConstructor = plugintype.getDeclaredConstructor() val noArgsConstructor = plugintype.getDeclaredConstructor()
noArgsConstructor.setAccessible(true) noArgsConstructor.setAccessible(true)
noArgsConstructor.newInstance() noArgsConstructor.newInstance()
...@@ -157,7 +157,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -157,7 +157,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
* Takes the QScripts passed in, runs their script() methods, retrieves their generated * Takes the QScripts passed in, runs their script() methods, retrieves their generated
* functions, and then builds and runs a QGraph based on the dependencies. * functions, and then builds and runs a QGraph based on the dependencies.
*/ */
def execute = { def execute: Int = {
var success = false var success = false
var result = 1 var result = 1
var functionsAndStatusSize = 0 var functionsAndStatusSize = 0
...@@ -181,7 +181,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -181,7 +181,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
} }
qGraph.messengers = qGraph.messengers =
allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger).toSeq allCommandPlugins.filter(_.statusMessenger != null).map(_.statusMessenger)
// TODO: Default command plugin argument? // TODO: Default command plugin argument?
val remoteFileConverter = val remoteFileConverter =
...@@ -270,7 +270,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -270,7 +270,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
* so that their arguments can be inspected before QScript.script is called. * so that their arguments can be inspected before QScript.script is called.
* @return Array of dynamic sources * @return Array of dynamic sources
*/ */
override def getArgumentSources = { override def getArgumentSources: Array[Class[_]] = {
var plugins = Seq.empty[Class[_]] var plugins = Seq.empty[Class[_]]
plugins ++= qScriptPluginManager.getPlugins plugins ++= qScriptPluginManager.getPlugins
plugins ++= qCommandPlugin.getPlugins plugins ++= qCommandPlugin.getPlugins
...@@ -281,7 +281,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -281,7 +281,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
* Returns the name of a script/plugin * Returns the name of a script/plugin
* @return The name of a script/plugin * @return The name of a script/plugin
*/ */
override def getArgumentSourceName(source: Class[_]) = { override def getArgumentSourceName(source: Class[_]): String = {
if (classOf[QScript].isAssignableFrom(source)) if (classOf[QScript].isAssignableFrom(source))
qScriptPluginManager.getName(source.asSubclass(classOf[QScript])) qScriptPluginManager.getName(source.asSubclass(classOf[QScript]))
else if (classOf[QCommandPlugin].isAssignableFrom(source)) else if (classOf[QCommandPlugin].isAssignableFrom(source))
...@@ -294,7 +294,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -294,7 +294,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
* Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections. * Returns a ScalaCompoundArgumentTypeDescriptor that can parse argument sources into scala collections.
* @return a ScalaCompoundArgumentTypeDescriptor * @return a ScalaCompoundArgumentTypeDescriptor
*/ */
override def getArgumentTypeDescriptors = override def getArgumentTypeDescriptors: util.List[ArgumentTypeDescriptor] =
util.Arrays.asList(new ScalaCompoundArgumentTypeDescriptor) util.Arrays.asList(new ScalaCompoundArgumentTypeDescriptor)
override def getApplicationDetails: ApplicationDetails = { override def getApplicationDetails: ApplicationDetails = {
...@@ -338,7 +338,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging { ...@@ -338,7 +338,7 @@ class BiopetQCommandLine extends CommandLineProgram with Logging {
} }
} }
def shutdown() = { def shutdown(): AnyVal = {
shuttingDown = true shuttingDown = true
qGraph.shutdown() qGraph.shutdown()
if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses) if (qScriptClasses != null) IOUtils.tryDelete(qScriptClasses)
......
...@@ -39,8 +39,8 @@ class BamStats(val parent: Configurable) ...@@ -39,8 +39,8 @@ class BamStats(val parent: Configurable)
var outputDir: File = _ var outputDir: File = _
var binSize: Option[Int] = config("bin_size") val binSize: Option[Int] = config("bin_size")
var threadBinSize: Option[Int] = config("thread_bin_size") val threadBinSize: Option[Int] = config("thread_bin_size")
override def defaultThreads = 3 override def defaultThreads = 3
override def defaultCoreMemory = 5.0 override def defaultCoreMemory = 5.0
...@@ -68,7 +68,7 @@ class BamStats(val parent: Configurable) ...@@ -68,7 +68,7 @@ class BamStats(val parent: Configurable)
} }
/** Creates command to execute extension */ /** Creates command to execute extension */
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("-b", bamFile) + required("-b", bamFile) +
required("-o", outputDir) + required("-o", outputDir) +
......
...@@ -43,8 +43,8 @@ class BastyGenerateFasta(val parent: Configurable) extends ToolCommandFunction w ...@@ -43,8 +43,8 @@ class BastyGenerateFasta(val parent: Configurable) extends ToolCommandFunction w
var snpsOnly: Boolean = config("snps_only", default = false) var snpsOnly: Boolean = config("snps_only", default = false)
var sampleName: String = _ var sampleName: String = _
var minAD: Int = config("min_ad", default = 8) val minAD: Int = config("min_ad", default = 8)
var minDepth: Int = config("min_depth", default = 8) val minDepth: Int = config("min_depth", default = 8)
var outputName: String = _ var outputName: String = _
override def defaultCoreMemory = 4.0 override def defaultCoreMemory = 4.0
...@@ -54,7 +54,7 @@ class BastyGenerateFasta(val parent: Configurable) extends ToolCommandFunction w ...@@ -54,7 +54,7 @@ class BastyGenerateFasta(val parent: Configurable) extends ToolCommandFunction w
reference = referenceFasta() reference = referenceFasta()
} }
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
optional("--inputVcf", inputVcf) + optional("--inputVcf", inputVcf) +
optional("--bamFile", bamFile) + optional("--bamFile", bamFile) +
......
...@@ -37,7 +37,7 @@ class BedToInterval(val parent: Configurable) extends ToolCommandFunction { ...@@ -37,7 +37,7 @@ class BedToInterval(val parent: Configurable) extends ToolCommandFunction {
override def defaultCoreMemory = 1.0 override def defaultCoreMemory = 1.0
override def cmdLine = override def cmdLine: String =
super.cmdLine + required("-I", input) + required("-b", bamFile) + required("-o", output) super.cmdLine + required("-I", input) + required("-b", bamFile) + required("-o", output)
} }
......
...@@ -31,7 +31,7 @@ class BedtoolsCoverageToCounts(val parent: Configurable) extends ToolCommandFunc ...@@ -31,7 +31,7 @@ class BedtoolsCoverageToCounts(val parent: Configurable) extends ToolCommandFunc
override def defaultCoreMemory = 3.0 override def defaultCoreMemory = 3.0
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("-I", input) + required("-I", input) +
required("-o", output) required("-o", output)
......
...@@ -37,7 +37,7 @@ class BiopetFlagstat(val parent: Configurable) extends ToolCommandFunction with ...@@ -37,7 +37,7 @@ class BiopetFlagstat(val parent: Configurable) extends ToolCommandFunction with
override def defaultCoreMemory = 6.0 override def defaultCoreMemory = 6.0
override def cmdLine = override def cmdLine: String =
super.cmdLine + required("-I", input) + required("-s", summaryFile) + " > " + required(output) super.cmdLine + required("-I", input) + required("-s", summaryFile) + " > " + required(output)
def summaryFiles: Map[String, File] = Map() def summaryFiles: Map[String, File] = Map()
......
...@@ -40,7 +40,7 @@ class CheckValidateAnnotation(val parent: Configurable) ...@@ -40,7 +40,7 @@ class CheckValidateAnnotation(val parent: Configurable)
var genomeName: String = "" var genomeName: String = ""
/** Exits whenever the input md5sum is not the same as the output md5sum */ /** Exits whenever the input md5sum is not the same as the output md5sum */
def run: Unit = { def run(): Unit = {
val reader = Source.fromFile(inputLogFile) val reader = Source.fromFile(inputLogFile)
reader.getLines().foreach { line => reader.getLines().foreach { line =>
......
...@@ -29,7 +29,7 @@ class DownloadNcbiAssembly(val parent: Configurable) extends ToolCommandFunction ...@@ -29,7 +29,7 @@ class DownloadNcbiAssembly(val parent: Configurable) extends ToolCommandFunction
var outputReport: File = _ var outputReport: File = _
@Input(required = true) @Input(required = true)
var assemblyReport: File = null var assemblyReport: File = _
var nameHeader: Option[String] = None var nameHeader: Option[String] = None
...@@ -38,7 +38,7 @@ class DownloadNcbiAssembly(val parent: Configurable) extends ToolCommandFunction ...@@ -38,7 +38,7 @@ class DownloadNcbiAssembly(val parent: Configurable) extends ToolCommandFunction
override def defaultCoreMemory = 4.0 override def defaultCoreMemory = 4.0
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("-a", assemblyReport) + required("-a", assemblyReport) +
required("--report", outputReport) + required("--report", outputReport) +
......
...@@ -36,7 +36,7 @@ class FastqSplitter(val parent: Configurable) extends ToolCommandFunction { ...@@ -36,7 +36,7 @@ class FastqSplitter(val parent: Configurable) extends ToolCommandFunction {
override def defaultCoreMemory = 4.0 override def defaultCoreMemory = 4.0
/** * Generate command to execute */ /** * Generate command to execute */
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("-I", input) + required("-I", input) +
repeat("-o", output) repeat("-o", output)
......
...@@ -35,31 +35,31 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa ...@@ -35,31 +35,31 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa
/** Original FASTQ file (read 1 or 2) */ /** Original FASTQ file (read 1 or 2) */
@Input(required = true) @Input(required = true)
var refFastq: File = null var refFastq: File = _
/** "Input read 1 FASTQ file" */ /** "Input read 1 FASTQ file" */
@Input(required = true) @Input(required = true)
var inputFastq1: File = null var inputFastq1: File = _
/** Input read 2 FASTQ file */ /** Input read 2 FASTQ file */
@Input(required = true) @Input(required = true)
var inputFastq2: File = null var inputFastq2: File = _
/** Output read 1 FASTQ file */ /** Output read 1 FASTQ file */
@Output(required = true) @Output(required = true)
var outputFastq1: File = null var outputFastq1: File = _
/** Output read 2 FASTQ file */ /** Output read 2 FASTQ file */
@Output(required = true) @Output(required = true)
var outputFastq2: File = null var outputFastq2: File = _
/** Sync statistics */ /** Sync statistics */
@Output(required = true) @Output(required = true)
var outputStats: File = null var outputStats: File = _
override def defaultCoreMemory = 4.0 override def defaultCoreMemory = 4.0
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("-r", refFastq) + required("-r", refFastq) +
required("-i", inputFastq1) + required("-i", inputFastq1) +
...@@ -98,7 +98,7 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa ...@@ -98,7 +98,7 @@ class FastqSync(val parent: Configurable) extends ToolCommandFunction with Summa
"num_reads_kept" -> countRLeft) "num_reads_kept" -> countRLeft)
} }
override def summaryDeps = outputStats :: super.summaryDeps override def summaryDeps: List[File] = outputStats :: super.summaryDeps
override def resolveSummaryConflict(v1: Any, v2: Any, key: String): Any = { override def resolveSummaryConflict(v1: Any, v2: Any, key: String): Any = {
(v1, v2) match { (v1, v2) match {
......
...@@ -63,7 +63,7 @@ class GensToVcf(val parent: Configurable) extends ToolCommandFunction with Refer ...@@ -63,7 +63,7 @@ class GensToVcf(val parent: Configurable) extends ToolCommandFunction with Refer
sortInput = true sortInput = true
} }
override def cmdLine = override def cmdLine: String =
super.cmdLine + super.cmdLine +
required("--inputGenotypes", inputGens) + required("--inputGenotypes", inputGens) +
required("--inputInfo", inputInfo) +