BiopetFifoPipe.scala 4.72 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
16 17 18 19 20
package nl.lumc.sasc.biopet.core

import java.io.File

import nl.lumc.sasc.biopet.utils.config.Configurable
21
import org.broadinstitute.gatk.utils.commandline.Output
Peter van 't Hof's avatar
Peter van 't Hof committed
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49

/**
 * Created by pjvan_thof on 9/29/15.
 */
class BiopetFifoPipe(val root: Configurable,
                     protected var commands: List[BiopetCommandLineFunction]) extends BiopetCommandLineFunction {

  def fifos: List[File] = {
    val outputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.outputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val inputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.inputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    for (
      cmdOutput <- commands;
      cmdInput <- commands if cmdOutput != cmdInput && outputs.contains(cmdOutput);
      outputFile <- outputs(cmdOutput) if inputs.contains(cmdInput);
      inputFile <- inputs(cmdInput) if outputFile == inputFile
    ) yield outputFile
  }

50 51 52
  @Output
  private var outputFiles: List[File] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
  override def beforeGraph(): Unit = {
    val outputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.outputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val inputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.inputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val fifoFiles = fifos

68 69 70
    outputFiles :::= outputs.values.toList.flatten.filter(!fifoFiles.contains(_))
    outputFiles = outputFiles.distinct

Peter van 't Hof's avatar
Peter van 't Hof committed
71
    deps :::= inputs.values.toList.flatten.filter(!fifoFiles.contains(_))
Peter van 't Hof's avatar
Peter van 't Hof committed
72
    deps = deps.distinct
73

Peter van 't Hof's avatar
Peter van 't Hof committed
74 75
    _pipesJobs :::= commands
    _pipesJobs = _pipesJobs.distinct
Peter van 't Hof's avatar
Peter van 't Hof committed
76 77

    analysisName = commands.map(_.analysisName).mkString("_")
Peter van 't Hof's avatar
Peter van 't Hof committed
78 79 80
  }

  override def beforeCmd(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
81
    commands.foreach { cmd =>
Peter van 't Hof's avatar
Peter van 't Hof committed
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
      cmd.beforeGraph()
      cmd.internalBeforeGraph()
      cmd.beforeCmd()
    }
  }

  def cmdLine = {
    val fifosFiles = this.fifos
    fifosFiles.filter(_.exists()).map(required("rm", _)).mkString("\n\n", " \n", " \n\n") +
      fifosFiles.map(required("mkfifo", _)).mkString("\n\n", "\n", "\n\n") +
      commands.map(_.commandLine).mkString("\n\n", " & \n", " & \n\n") +
      BiopetFifoPipe.waitScript +
      fifosFiles.map(required("rm", _)).mkString("\n\n", " \n", " \n\n") +
      BiopetFifoPipe.endScript
  }
97 98 99 100

  override def setResources(): Unit = {
    combineResources(commands)
  }
101 102 103 104 105 106

  override def setupRetry(): Unit = {
    super.setupRetry()
    commands.foreach(_.setupRetry())
    combineResources(commands)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
107 108 109 110 111

  override def freezeFieldValues(): Unit = {
    super.freezeFieldValues()
    commands.foreach(_.qSettings = qSettings)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
112 113 114 115 116
}

object BiopetFifoPipe {
  val waitScript =
    """
117 118 119 120 121
      |
      |allJobs=`jobs -p`
      |jobs=$allJobs
      |
      |echo [`date`] pids: $jobs
Peter van 't Hof's avatar
Peter van 't Hof committed
122 123 124
      |
      |FAIL="0"
      |
Peter van 't Hof's avatar
Peter van 't Hof committed
125
      |while echo $jobs | grep -e "\w" > /dev/null
Peter van 't Hof's avatar
Peter van 't Hof committed
126
      |do
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
      |	for job in $jobs
      |	do
      |		if ps | grep "$job " | grep -v grep > /dev/null
      |		then
      |		    echo [`date`] $job still running > /dev/null
      |		else
      |		    jobs=`echo $jobs | sed "s/${job}//"`
      |			wait $job || FAIL=$?
      |			if echo $FAIL | grep -ve "^0$" > /dev/null
      |			then
      |			    echo [`date`] $job fails with exitcode: $FAIL
      |				break
      |			fi
      |			echo [`date`] $job done
      |		fi
      |	done
      |	if echo $FAIL | grep -ve "^0$" > /dev/null
      |    then
      |        break
      |    fi
      |	sleep 1
Peter van 't Hof's avatar
Peter van 't Hof committed
148 149
      |done
      |
150 151 152 153 154 155 156 157
      |if echo $FAIL | grep -ve "^0$" > /dev/null
      |then
      |    echo [`date`] kill other pids: $jobs
      |    kill $jobs
      |fi
      |
      |echo [`date`] Done
      |
Peter van 't Hof's avatar
Peter van 't Hof committed
158 159 160 161 162 163 164 165
      |
    """.stripMargin

  val endScript =
    """
      |
      |if [ "$FAIL" == "0" ];
      |then
166
      |echo [`date`] "BiopetFifoPipe Done"
Peter van 't Hof's avatar
Peter van 't Hof committed
167
      |else
168
      |echo [`date`] BiopetFifoPipe "FAIL! ($FAIL)"
Peter van 't Hof's avatar
Peter van 't Hof committed
169 170 171 172 173 174
      |exit $FAIL
      |fi
      |
      |
    """.stripMargin
}