GearsReport.scala

/**
  * Biopet is built on top of GATK Queue for building bioinformatic
  * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
  * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
  * should also be able to execute Biopet tools and pipelines.
  *
  * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
  *
  * Contact us at: sasc@lumc.nl
  *
  * A dual licensing mode is applied. The source code within this project is freely available for non-commercial use under an AGPL
  * license; For commercial users or users who do not want to follow the AGPL
  * license, please contact us to obtain a separate license.
  */
package nl.lumc.sasc.biopet.pipelines.gears

import java.io.File

import nl.lumc.sasc.biopet.core.report.{
  MultisampleReportBuilder,
  ReportBuilderExtension,
  ReportPage,
  ReportSection
}
import nl.lumc.sasc.biopet.pipelines.flexiprep.{
  FlexiprepBaseSummary,
  FlexiprepReadSummary,
  FlexiprepReport
}
import nl.lumc.sasc.biopet.pipelines.gears
import nl.lumc.sasc.biopet.utils.config.Configurable
import nl.lumc.sasc.biopet.utils.summary.db.Schema.{Library, Sample}
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.Implicts._
import nl.lumc.sasc.biopet.utils.summary.db.SummaryDb.{ModuleName, NoLibrary, NoModule, SampleId}

import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}

/**
  * Report for Gears
  *
  * Created by pjvan_thof on 12/10/15.
  */
class GearsReport(val parent: Configurable) extends ReportBuilderExtension {
  def builder = GearsReport
}

object GearsReport extends MultisampleReportBuilder {

  def pipelineName = "gears"

  def reportName = "Gears Report"

  override def extFiles: List[gears.GearsReport.ExtFile] =
    super.extFiles ++ List("js/gears.js",
                           "js/krona-2.0.js",
                           "img/krona/loading.gif",
                           "img/krona/hidden.png",
                           "img/krona/favicon.ico")
      .map(x => ExtFile("/nl/lumc/sasc/biopet/pipelines/gears/report/ext/" + x, x))

  def indexPage: Future[ReportPage] = {
    val run = Await.result(summary.getRuns(runId).map(_.head), Duration.Inf)

    val krakenExecuted = summary.getStatsSize(runId = runId,
                                              pipeline = "gearskraken",
                                              module = "krakenreport",
                                              library = NoLibrary,
                                              mustHaveSample = true) >= samples.size
    val centrifugeExecuted = summary.getStatsSize(runId,
                                                  "gearscentrifuge",
                                                  "centrifuge_report",
                                                  library = NoLibrary,
                                                  mustHaveSample = true) >= samples.size
    val qiimeClosesOtuTable = summary.getFile(runId, "gears", key = "qiime_closed_otu_table")
    val qiimeOpenOtuTable = summary.getFile(runId, "gears", key = "qiime_open_otu_table")
    val centrifugePageAllMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                   runId,
                                                                   "gearscentrifuge",
                                                                   "centrifuge_report",
                                                                   samples,
                                                                   libraries,
                                                                   sampleId,
                                                                   libId,
                                                                   Some("centrifuge"))
    val centrifugePageUniqueMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                      runId,
                                                                      "gearscentrifuge",
                                                                      "centrifuge_unique_report",
                                                                      samples,
                                                                      libraries,
                                                                      sampleId,
                                                                      libId)

    val centrifugePage =
      if (centrifugeExecuted)
        Some(
          "Centrifuge analysis" -> Future.successful(ReportPage(
            List(
              "Non-unique" ->
                Future.successful(
                  ReportPage(
                    List(),
                    List("All mappings" -> ReportSection(
                      "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                      centrifugePageAllMappingsKronaPlot)),
                    Map()
                  ))),
            List("Unique mappings" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
              centrifugePageUniqueMappingsKronaPlot)),
            Map("summaryModuleTag" -> "gearscentrifuge", "centrifugeTag" -> Some("centrifuge"))
          )))
      else None
    val krakenPageKronaPlot = GearsKronaPlot
      .values(summary, runId, "gearskraken", "krakenreport", samples, libraries, sampleId, libId)
    val krakenPage =
      if (krakenExecuted)
        Some(
          "Kraken analysis" -> Future.successful(
            ReportPage(
              List(),
              List("Krona plot" -> Future.successful(
                ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                              krakenPageKronaPlot))),
              Map()
            )))
      else None

    val qiimeClosedPage =
      if (qiimeClosesOtuTable.isDefined)
        Some(
          "Qiime closed reference analysis" -> Future.successful(ReportPage(
            List(),
            List("Krona plot" -> Future.successful(
              ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp"))),
            Map("biomFile" -> new File(
              run.outputDir + File.separator + qiimeClosesOtuTable.get.path))
          )))
      else None

    val qiimeOpenPage =
      if (qiimeOpenOtuTable.isDefined)
        Some(
          "Qiime open reference analysis" -> Future.successful(ReportPage(
            List(),
            List("Krona plot" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp")),
            Map(
              "biomFile" -> new File(run.outputDir + File.separator + qiimeOpenOtuTable.get.path))
          )))
      else None

    val flexiprepReadSummary =
      FlexiprepReadSummary.values(summary,
                                  runId,
                                  samples,
                                  libraries,
                                  sampleId,
                                  libId,
                                  showPlot = true,
                                  showTable = false)
    val flexiprepBaseSummary =
      FlexiprepBaseSummary.values(summary,
                                  runId,
                                  samples,
                                  libraries,
                                  sampleId,
                                  libId,
                                  showPlot = true,
                                  showTable = false)
    Future {
      ReportPage(
        List(centrifugePage, krakenPage, qiimeClosedPage, qiimeOpenPage).flatten ::: List(
          "Samples" -> generateSamplesPage(pageArgs)
        ),
        List("Report" -> ReportSection("/nl/lumc/sasc/biopet/pipelines/gears/gearsFront.ssp")) ++
          List(
            "QC reads" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepReadSummary.ssp",
              flexiprepReadSummary),
            "QC bases" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepBaseSummary.ssp",
              flexiprepBaseSummary)
          ),
        pageArgs
      )
    }
  }

  /** Single sample page */
  def samplePage(sampleId: Int, args: Map[String, Any]): Future[ReportPage] = {
    val run = Await.result(summary.getRuns(runId).map(_.head), Duration.Inf)
    val krakenExecuted = Await.result(summary.getStatsSize(runId,
                                                           "gearskraken",
                                                           "krakenreport",
                                                           sample = sampleId,
                                                           library = NoLibrary),
                                      Duration.Inf) == 1
    val centrifugeExecuted = Await.result(summary.getStatsSize(runId,
                                                               "gearscentrifuge",
                                                               "centrifuge_report",
                                                               sample = sampleId,
                                                               library = None),
                                          Duration.Inf) == 1
    val qiimeClosesOtuTable = Await.result(summary.getFile(runId,
                                                           "gearssingle",
                                                           NoModule,
                                                           sampleId,
                                                           NoLibrary,
                                                           "qiime_closed_otu_table"),
                                           Duration.Inf)
    val qiimeOpenOtuTable = Await.result(
      summary.getFile(runId, "gearssingle", NoModule, sampleId, NoLibrary, "qiime_open_otu_table"),
      Duration.Inf)
    val centrifugePageAllMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                   runId,
                                                                   "gearscentrifuge",
                                                                   "centrifuge_report",
                                                                   samples,
                                                                   libraries,
                                                                   sampleId,
                                                                   libId,
                                                                   Some("centrifuge"))
    val centrifugePageUniqueMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                      runId,
                                                                      "gearscentrifuge",
                                                                      "centrifuge_unique_report",
                                                                      samples,
                                                                      libraries,
                                                                      sampleId,
                                                                      libId)
    val gearsCentrifugePage = if (centrifugeExecuted) {
      Some(
        "Centrifuge analysis" -> Future.successful(ReportPage(
          List(
            "Non-unique" -> Future.successful(
              ReportPage(
                List(),
                List("All mappings" -> ReportSection(
                  "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                  centrifugePageAllMappingsKronaPlot
                )),
                Map()
              ))),
          List("Unique mappings" -> ReportSection(
            "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
            centrifugePageUniqueMappingsKronaPlot)),
          Map("summaryModuleTag" -> "gearscentrifuge", "centrifugeTag" -> Some("centrifuge"))
        )))
    } else None
    val krakenPageKronaPlot = GearsKronaPlot
      .values(summary, runId, "gearskraken", "krakenreport", samples, libraries, sampleId, libId)
    val krakenAnalysisPage =
      if (krakenExecuted)
        Some(
          "Kraken analysis" -> Future.successful(
            ReportPage(
              List(),
              List("Krona plot" -> ReportSection(
                "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                krakenPageKronaPlot)),
              Map()
            )))
      else None

    val qiimeClosesOtuTablePage =
      if (qiimeClosesOtuTable.isDefined)
        Some(
          "Qiime closed reference analysis" -> Future.successful(ReportPage(
            List(),
            List("Krona plot" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp")),
            Map("biomFile" -> new File(
              run.outputDir + File.separator + qiimeClosesOtuTable.get.path))
          )))
      else None

    val qiimeOpenOtuTablePage =
      if (qiimeOpenOtuTable.isDefined)
        Some(
          "Qiime open reference analysis" -> Future.successful(ReportPage(
            List(),
            List("Krona plot" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp")),
            Map(
              "biomFile" -> new File(run.outputDir + File.separator + qiimeOpenOtuTable.get.path))
          )))
      else None

    val flexiprepReadSummary =
      FlexiprepReadSummary.values(summary, runId, samples, libraries, sampleId)
    val flexiprepBaseSummary =
      FlexiprepBaseSummary.values(summary, runId, samples, libraries, sampleId)
    Future {
      ReportPage(
        subPages =
          List(gearsCentrifugePage,
               krakenAnalysisPage,
               qiimeClosesOtuTablePage,
               qiimeOpenOtuTablePage).flatten ::: List(
            "Libraries" -> generateLibraryPage(args)
          ),
        sections = List(
          "QC reads" -> ReportSection(
            "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepReadSummary.ssp",
            flexiprepReadSummary),
          "QC bases" -> ReportSection(
            "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepBaseSummary.ssp",
            flexiprepBaseSummary)
        ),
        args = args
      )
    }
  }

  /** Library page */
  def libraryPage(sampleId: Int, libId: Int, args: Map[String, Any]): Future[ReportPage] = {
    val sName = Await.result(summary.getSampleName(sampleId), Duration.Inf)
    val lName = Await.result(summary.getLibraryName(libId), Duration.Inf)

    val flexiprepExecuted = Await.result(
      summary.getStatsSize(runId, "flexiprep", sample = sampleId, library = libId),
      Duration.Inf) >= 1

    val krakenExecuted = Await.result(summary.getStatsSize(runId,
                                                           "gearskraken",
                                                           "krakenreport",
                                                           sample = sampleId,
                                                           library = libId),
                                      Duration.Inf) == 1
    val centrifugeExecuted = Await.result(summary.getStatsSize(runId,
                                                               "gearscentrifuge",
                                                               "centrifuge_report",
                                                               sample = sampleId,
                                                               library = libId),
                                          Duration.Inf) == 1
    val qiimeClosesOtuTable = Await.result(
      summary.getFile(runId, "gears", NoModule, sampleId, libId, "qiime_closed_otu_table"),
      Duration.Inf)
    val qiimeOpenOtuTable = Await.result(
      summary.getFile(runId, "gears", NoModule, sampleId, libId, "qiime_open_otu_table"),
      Duration.Inf)

    val flexiprepReportPage =
      if (flexiprepExecuted) {
        val flexiprepPageSummaries = FlexiprepReport.flexiprepPageSummaries(summary,sampleId,libId)
        Some("QC" -> FlexiprepReport.flexiprepPage(flexiprepPageSummaries))
      }
      else None

    val centrifugePageAllMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                   runId,
                                                                   "gearscentrifuge",
                                                                   "centrifuge_report",
                                                                   samples,
                                                                   libraries,
                                                                   sampleId,
                                                                   libId,
                                                                   Some("centrifuge"))
    val centrifugePageUniqueMappingsKronaPlot = GearsKronaPlot.values(summary,
                                                                      runId,
                                                                      "gearscentrifuge",
                                                                      "centrifuge_unique_report",
                                                                      samples,
                                                                      libraries,
                                                                      sampleId,
                                                                      libId)
    val centrifugeReportPage =
      if (centrifugeExecuted)
        Some(
          "Centrifuge analysis" -> Future.successful(ReportPage(
            List(
              "Non-unique" -> Future.successful(
                ReportPage(
                  List(),
                  List("All mappings" -> ReportSection(
                    "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                    centrifugePageAllMappingsKronaPlot
                  )),
                  Map()
                ))),
            List("Unique mappings" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
              centrifugePageUniqueMappingsKronaPlot)),
            Map("summaryModuleTag" -> "gearscentrifuge", "centrifugeTag" -> Some("centrifuge"))
          )))
      else None
    val krakenPageKronaPlot = GearsKronaPlot
      .values(summary, runId, "gearskraken", "krakenreport", samples, libraries, sampleId, libId)
    val krakenAnalysisPage =
      if (krakenExecuted)
        Some(
          "Kraken analysis" -> Future.successful(
            ReportPage(
              List(),
              List("Krona plot" -> ReportSection(
                "/nl/lumc/sasc/biopet/pipelines/gears/krakenKrona.ssp",
                krakenPageKronaPlot)),
              Map()
            )))
      else None

    val qiimeClosesOtuTablePage =
      if (qiimeClosesOtuTable.isDefined)
        Some(
          "Qiime closed reference analysis" -> Future.successful(ReportPage(
            List(),
            List("Krona plot" -> ReportSection(
              "/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp")),
            Map("biomFile" -> new File(qiimeClosesOtuTable.get.path))
          )))
      else None

    val qiimeOpenOtuTablePage =
      if (qiimeOpenOtuTable.isDefined)
        Some(
          "Qiime open reference analysis" -> Future.successful(
            ReportPage(List(),
                       List("Krona plot" -> ReportSection(
                         "/nl/lumc/sasc/biopet/pipelines/gears/qiimeKrona.ssp")),
                       Map("biomFile" -> new File(qiimeOpenOtuTable.get.path)))))
      else None
    val flexiprepReadSummary =
      FlexiprepReadSummary.values(summary, runId, samples, libraries, sampleId)
    val flexiprepBaseSummary =
      FlexiprepBaseSummary.values(summary, runId, samples, libraries, sampleId)
    Future {
      ReportPage(
        List(flexiprepReportPage,
             centrifugeReportPage,
             krakenAnalysisPage,
             qiimeClosesOtuTablePage,
             qiimeOpenOtuTablePage).flatten,
        List(
          "QC reads" -> ReportSection(
            "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepReadSummary.ssp",
            flexiprepReadSummary),
          "QC bases" -> ReportSection(
            "/nl/lumc/sasc/biopet/pipelines/flexiprep/flexiprepBaseSummary.ssp",
            flexiprepBaseSummary)
        ),
        args
      )
    }
  }

}

object GearsKronaPlot {
  def values(summary: SummaryDb,
             runId: Int,
             summaryPipelineName: String,
             summaryModuleName: String,
             allSamples: Seq[Sample],
             allLibraries: Seq[Library],
             sampleId: Option[Int] = None,
             libId: Option[Int] = None,
             centrifugeTag: Option[String] = None): Map[String, Any] = {
    val summariesVal =
      summaries(summary, runId, sampleId, libId, summaryPipelineName, summaryModuleName)
    val totalReadsVal = totalReads(summary,
                                   runId,
                                   sampleId,
                                   libId,
                                   summaryPipelineName,
                                   centrifugeTag,
                                   allSamples,
                                   allLibraries)
    Map(
      "summary" -> summary,
      "runId" -> runId,
      "sampleId" -> sampleId,
      "libId" -> libId,
      "summaryPipelineName" -> summaryPipelineName,
      "summaryModuleName" -> summaryModuleName,
      "centrifugeTag" -> centrifugeTag,
      "allSamples" -> allSamples,
      "allLibraries" -> allLibraries,
      "summaries" -> summariesVal,
      "totalReads" -> totalReadsVal
    )
  }

  def summaries(summary: SummaryDb,
                runId: Int,
                sampleId: Option[Int],
                libId: Option[Int],
                summaryPipelineName: String,
                summaryModuleName: String): Map[Int, Map[String, Option[Any]]] = {
    if (libId.isDefined)
      summary
        .getStatsForLibraries(runId,
                              summaryPipelineName,
                              summaryModuleName,
                              sampleId,
                              Map("all" -> Nil))
        .filter(_._1._2 == libId.get)
        .map(x => x._1._1 -> x._2)
    else
      summary.getStatsForSamples(runId,
                                 summaryPipelineName,
                                 summaryModuleName,
                                 sampleId.map(SampleId),
                                 Map("all" -> Nil))
  }
  def totalReads(
      summary: SummaryDb,
      runId: Int,
      sampleId: Option[Int],
      libId: Option[Int],
      summaryPipelineName: String,
      centrifugeTag: Option[String],
      allSamples: Seq[Sample],
      allLibraries: Seq[Library]
  ): Option[Map[String, Long]] = {
    centrifugeTag.map { tag =>
      if (libId.isDefined) {
        val stats = summary
          .getStatsForLibraries(runId,
                                summaryPipelineName,
                                ModuleName(tag),
                                sampleId,
                                Map("total" -> List("metrics", "Read")))
          .filter(_._1._2 == libId.get)
          .head
        val lib = allLibraries.filter(_.id == stats._1._2).head
        val sample = allSamples.filter(_.id == stats._1._1).head
        Map(s"${sample.name}" -> stats._2("total").map(_.toString.toLong).getOrElse(0L))
      } else
        summary
          .getStatsForSamples(runId,
                              summaryPipelineName,
                              ModuleName(tag),
                              sampleId.map(SummaryDb.SampleId),
                              Map("total" -> List("metrics", "Read")))
          .map(
            x =>
              allSamples
                .find(_.id == x._1)
                .head
                .name -> x._2("total").map(_.toString.toLong).getOrElse(0L))
    }
  }

}