Commit 877cb631 authored by bow's avatar bow
Browse files

Refactor Processors base classes

parent b4a35669
...@@ -17,17 +17,19 @@ ...@@ -17,17 +17,19 @@
package nl.lumc.sasc.sentinel.api package nl.lumc.sasc.sentinel.api
import java.io.File import java.io.File
import scala.util.{ Failure, Success, Try }
import scala.util.{ Failure, Success }
import org.scalatra._ import org.scalatra._
import org.scalatra.swagger._ import org.scalatra.swagger._
import org.scalatra.servlet.{ FileItem, FileUploadSupport, MultipartConfig, SizeConstraintExceededException } import org.scalatra.servlet.{ FileUploadSupport, MultipartConfig, SizeConstraintExceededException }
import nl.lumc.sasc.sentinel.{ AllowedPipelineParams, HeaderApiKey, Pipeline } import nl.lumc.sasc.sentinel.{ AllowedPipelineParams, HeaderApiKey, Pipeline }
import nl.lumc.sasc.sentinel.api.auth.AuthenticationSupport import nl.lumc.sasc.sentinel.api.auth.AuthenticationSupport
import nl.lumc.sasc.sentinel.db._ import nl.lumc.sasc.sentinel.db._
import nl.lumc.sasc.sentinel.processors.gentrap.GentrapV04InputProcessor import nl.lumc.sasc.sentinel.processors.gentrap.GentrapV04RunsProcessor
import nl.lumc.sasc.sentinel.processors.plain.PlainInputProcessor import nl.lumc.sasc.sentinel.processors.plain.PlainRunsProcessor
import nl.lumc.sasc.sentinel.processors.GenericRunsProcessor
import nl.lumc.sasc.sentinel.settings._ import nl.lumc.sasc.sentinel.settings._
import nl.lumc.sasc.sentinel.models._ import nl.lumc.sasc.sentinel.models._
import nl.lumc.sasc.sentinel.utils._ import nl.lumc.sasc.sentinel.utils._
...@@ -50,19 +52,16 @@ class RunsController(implicit val swagger: Swagger, mongo: MongodbAccessObject) ...@@ -50,19 +52,16 @@ class RunsController(implicit val swagger: Swagger, mongo: MongodbAccessObject)
protected val applicationDescription: String = "Submission and retrieval of run summaries" protected val applicationDescription: String = "Submission and retrieval of run summaries"
/** Adapter for connecting to run collections. */ /** Adapter for connecting to run collections. */
val runs = new RunsAdapter { val runs = new GenericRunsProcessor(mongo)
val mongo = self.mongo
def processRun(fi: FileItem, user: User, pipeline: Pipeline.Value) = Try(throw new NotImplementedError)
}
/** Adapter for connecting to users collection. */ /** Adapter for connecting to users collection. */
val users = new UsersAdapter { val mongo = self.mongo } val users = new UsersAdapter { val mongo = self.mongo }
/** Adapter for connecting to the plain summary collections. */ /** Adapter for connecting to the plain summary collections. */
val plain = new PlainInputProcessor(mongo) val plain = new PlainRunsProcessor(mongo)
/** Adapter for connecting to the gentrap summary collections. */ /** Adapter for connecting to the gentrap summary collections. */
val gentrap = new GentrapV04InputProcessor(mongo) val gentrap = new GentrapV04RunsProcessor(mongo)
/** Set maximum allowed file upload size. */ /** Set maximum allowed file upload size. */
configureMultipartHandling(MultipartConfig(maxFileSize = Some(MaxRunSummarySize))) configureMultipartHandling(MultipartConfig(maxFileSize = Some(MaxRunSummarySize)))
......
...@@ -21,6 +21,7 @@ import com.novus.salat._ ...@@ -21,6 +21,7 @@ import com.novus.salat._
import com.novus.salat.global._ import com.novus.salat.global._
import nl.lumc.sasc.sentinel.models.{ BaseLibRecord, BaseSampleRecord } import nl.lumc.sasc.sentinel.models.{ BaseLibRecord, BaseSampleRecord }
import nl.lumc.sasc.sentinel.processors.RunsProcessor
/** /**
* Trait for storing samples and libraries from run summaries. * Trait for storing samples and libraries from run summaries.
...@@ -28,10 +29,7 @@ import nl.lumc.sasc.sentinel.models.{ BaseLibRecord, BaseSampleRecord } ...@@ -28,10 +29,7 @@ import nl.lumc.sasc.sentinel.models.{ BaseLibRecord, BaseSampleRecord }
* @tparam S Subclass of [[nl.lumc.sasc.sentinel.models.BaseSampleRecord]] representing a sample run by a pipeline. * @tparam S Subclass of [[nl.lumc.sasc.sentinel.models.BaseSampleRecord]] representing a sample run by a pipeline.
* @tparam L Subclass of [[nl.lumc.sasc.sentinel.models.BaseLibRecord]] representing a library run by a pipeline. * @tparam L Subclass of [[nl.lumc.sasc.sentinel.models.BaseLibRecord]] representing a library run by a pipeline.
*/ */
trait UnitsAdapter[S <: BaseSampleRecord, L <: BaseLibRecord] extends MongodbConnector { this: RunsAdapter => trait UnitsAdapter[S <: BaseSampleRecord, L <: BaseLibRecord] extends MongodbConnector { this: RunsProcessor =>
/** Name of the pipeline that produces the run summary file. */
def pipelineName: String
/** Collection for the samples. */ /** Collection for the samples. */
private lazy val samplesColl = mongo.db(collectionNames.pipelineSamples(pipelineName)) private lazy val samplesColl = mongo.db(collectionNames.pipelineSamples(pipelineName))
......
/*
* Copyright (c) 2015 Leiden University Medical Center and contributors
* (see AUTHORS.md file for details).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.lumc.sasc.sentinel.processors
import scala.util.Try
import org.scalatra.servlet.FileItem
import nl.lumc.sasc.sentinel.Pipeline
import nl.lumc.sasc.sentinel.db.MongodbAccessObject
import nl.lumc.sasc.sentinel.models._
/**
* Input processor that does not process incoming runs. Rather, it is used for querying run data of any pipelines.
*/
class GenericRunsProcessor(mongo: MongodbAccessObject) extends RunsProcessor(mongo) {
def pipelineName = "generic"
def processRun(fi: FileItem, user: User, pipeline: Pipeline.Value) = Try(throw new NotImplementedError)
}
/*
* Copyright (c) 2015 Leiden University Medical Center and contributors
* (see AUTHORS.md file for details).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.lumc.sasc.sentinel.processors
import nl.lumc.sasc.sentinel.db.{ MongodbAccessObject, MongodbConnector }
/** Base trait for defining processors. */
trait Processor extends MongodbConnector {
/** Run summary pipeline name. */
def pipelineName: String
}
...@@ -14,23 +14,26 @@ ...@@ -14,23 +14,26 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package nl.lumc.sasc.sentinel.db package nl.lumc.sasc.sentinel.processors
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import scala.util.Try
import com.mongodb.casbah.gridfs.GridFSDBFile
import com.mongodb.casbah.Imports._ import com.mongodb.casbah.Imports._
import com.mongodb.casbah.gridfs.GridFSDBFile
import com.novus.salat._ import com.novus.salat._
import com.novus.salat.global._ import com.novus.salat.global._
import org.scalatra.servlet.FileItem
import nl.lumc.sasc.sentinel.Pipeline import nl.lumc.sasc.sentinel.Pipeline
import nl.lumc.sasc.sentinel.db.MongodbAccessObject
import nl.lumc.sasc.sentinel.models.{ PipelineStats, RunRecord, User } import nl.lumc.sasc.sentinel.models.{ PipelineStats, RunRecord, User }
import nl.lumc.sasc.sentinel.utils.{ DuplicateFileException, calcMd5, getUtcTimeNow } import nl.lumc.sasc.sentinel.utils.{ DuplicateFileException, calcMd5, getUtcTimeNow }
import org.scalatra.servlet.FileItem
import scala.util.Try
/** Trait for processing and storing run summary files to a run records collection */ /**
trait RunsAdapter extends MongodbConnector { * Base class for processing run summary files.
*/
abstract class RunsProcessor(protected val mongo: MongodbAccessObject) extends Processor {
/** /**
* Processes and stores the given uploaded file to the run records collection. * Processes and stores the given uploaded file to the run records collection.
......
...@@ -23,20 +23,17 @@ import com.novus.salat._ ...@@ -23,20 +23,17 @@ import com.novus.salat._
import com.novus.salat.global._ import com.novus.salat.global._
import com.mongodb.casbah.Imports._ import com.mongodb.casbah.Imports._
import nl.lumc.sasc.sentinel.{ AccLevel, LibType, SeqQcPhase } import nl.lumc.sasc.sentinel.{ AccLevel, LibType }
import nl.lumc.sasc.sentinel.db.MongodbConnector import nl.lumc.sasc.sentinel.db.MongodbAccessObject
import nl.lumc.sasc.sentinel.models.{ SeqStatsAggr, User } import nl.lumc.sasc.sentinel.models.{ SeqStatsAggr, User }
/** /**
* Base trait that provides support for querying and aggregating metrics for a pipeline. * Base class that provides support for querying and aggregating statistics for a pipeline.
*/ */
trait OutputProcessor extends MongodbConnector { abstract class StatsProcessor(protected val mongo: MongodbAccessObject) extends Processor {
// TODO: refactor functions in here ~ we can do with less duplication // TODO: refactor functions in here ~ we can do with less duplication
/** Name of the pipeline that produces the metrics to query. */
def pipelineName: String
/** Name of the unit attribute that denotes whether it comes from a paired-end library or not. */ /** Name of the unit attribute that denotes whether it comes from a paired-end library or not. */
protected implicit val pairAttrib = "isPaired" protected implicit val pairAttrib = "isPaired"
......
...@@ -18,14 +18,14 @@ package nl.lumc.sasc.sentinel.processors.gentrap ...@@ -18,14 +18,14 @@ package nl.lumc.sasc.sentinel.processors.gentrap
import nl.lumc.sasc.sentinel.db.MongodbAccessObject import nl.lumc.sasc.sentinel.db.MongodbAccessObject
import nl.lumc.sasc.sentinel.models._ import nl.lumc.sasc.sentinel.models._
import nl.lumc.sasc.sentinel.processors.OutputProcessor import nl.lumc.sasc.sentinel.processors.StatsProcessor
/** /**
* Output processor for Gentrap endpoints. * Output processor for Gentrap endpoints.
* *
* @param mongo MongoDB database access object. * @param mongo MongoDB database access object.
*/ */
class GentrapOutputProcessor(protected val mongo: MongodbAccessObject) extends OutputProcessor { class GentrapStatsProcessor(mongo: MongodbAccessObject) extends StatsProcessor(mongo) {
def pipelineName = "gentrap" def pipelineName = "gentrap"
......
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
*/ */
package nl.lumc.sasc.sentinel.processors.gentrap package nl.lumc.sasc.sentinel.processors.gentrap
import nl.lumc.sasc.sentinel.processors.RunsProcessor
import scala.util.Try import scala.util.Try
import org.apache.commons.io.FilenameUtils.{ getExtension, getName } import org.apache.commons.io.FilenameUtils.{ getExtension, getName }
...@@ -37,9 +39,9 @@ import nl.lumc.sasc.sentinel.validation.ValidationAdapter ...@@ -37,9 +39,9 @@ import nl.lumc.sasc.sentinel.validation.ValidationAdapter
* *
* @param mongo MongoDB database access object. * @param mongo MongoDB database access object.
*/ */
class GentrapV04InputProcessor(protected val mongo: MongodbAccessObject) class GentrapV04RunsProcessor(mongo: MongodbAccessObject)
extends UnitsAdapter[GentrapSampleRecord, GentrapLibRecord] extends RunsProcessor(mongo)
with RunsAdapter with UnitsAdapter[GentrapSampleRecord, GentrapLibRecord]
with ValidationAdapter with ValidationAdapter
with ReferencesAdapter with ReferencesAdapter
with AnnotationsAdapter { with AnnotationsAdapter {
......
...@@ -19,6 +19,7 @@ package nl.lumc.sasc.sentinel.processors.plain ...@@ -19,6 +19,7 @@ package nl.lumc.sasc.sentinel.processors.plain
import java.time.Clock import java.time.Clock
import java.util.Date import java.util.Date
import nl.lumc.sasc.sentinel.models.{ RunRecord, User } import nl.lumc.sasc.sentinel.models.{ RunRecord, User }
import nl.lumc.sasc.sentinel.processors.RunsProcessor
import scala.util.Try import scala.util.Try
import org.scalatra.servlet.FileItem import org.scalatra.servlet.FileItem
...@@ -37,10 +38,11 @@ import nl.lumc.sasc.sentinel.validation.ValidationAdapter ...@@ -37,10 +38,11 @@ import nl.lumc.sasc.sentinel.validation.ValidationAdapter
* *
* @param mongo MongoDB database access object. * @param mongo MongoDB database access object.
*/ */
class PlainInputProcessor(protected val mongo: MongodbAccessObject) class PlainRunsProcessor(mongo: MongodbAccessObject) extends RunsProcessor(mongo)
extends RunsAdapter
with ValidationAdapter { with ValidationAdapter {
def pipelineName = "plain"
val validator = createValidator("/schemas/plain.json") val validator = createValidator("/schemas/plain.json")
def processRun(fi: FileItem, user: User, pipeline: Pipeline.Value) = def processRun(fi: FileItem, user: User, pipeline: Pipeline.Value) =
......
...@@ -28,7 +28,7 @@ class GentrapValidationSpec extends Specification with JsonLoader with Mockito { ...@@ -28,7 +28,7 @@ class GentrapValidationSpec extends Specification with JsonLoader with Mockito {
"Support for the 'gentrap' pipeline" should { "Support for the 'gentrap' pipeline" should {
br br
val ipv04 = new GentrapV04InputProcessor(mongo) val ipv04 = new GentrapV04RunsProcessor(mongo)
"exclude non-gentrap summary files" in { "exclude non-gentrap summary files" in {
val summary = loadJson("/schema_examples/plain.json") val summary = loadJson("/schema_examples/plain.json")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment