BiopetFifoPipe.scala 4.72 KB
Newer Older
Peter van 't Hof's avatar
Peter van 't Hof committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * Biopet is built on top of GATK Queue for building bioinformatic
 * pipelines. It is mainly intended to support LUMC SHARK cluster which is running
 * SGE. But other types of HPC that are supported by GATK Queue (such as PBS)
 * should also be able to execute Biopet tools and pipelines.
 *
 * Copyright 2014 Sequencing Analysis Support Core - Leiden University Medical Center
 *
 * Contact us at: sasc@lumc.nl
 *
 * A dual licensing mode is applied. The source code within this project that are
 * not part of GATK Queue is freely available for non-commercial use under an AGPL
 * license; For commercial users or users who do not want to follow the AGPL
 * license, please contact us to obtain a separate license.
 */
Peter van 't Hof's avatar
Peter van 't Hof committed
16
17
18
19
20
package nl.lumc.sasc.biopet.core

import java.io.File

import nl.lumc.sasc.biopet.utils.config.Configurable
21
import org.broadinstitute.gatk.utils.commandline.Output
Peter van 't Hof's avatar
Peter van 't Hof committed
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

/**
 * Created by pjvan_thof on 9/29/15.
 */
class BiopetFifoPipe(val root: Configurable,
                     protected var commands: List[BiopetCommandLineFunction]) extends BiopetCommandLineFunction {

  def fifos: List[File] = {
    val outputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.outputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val inputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.inputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    for (
      cmdOutput <- commands;
      cmdInput <- commands if cmdOutput != cmdInput && outputs.contains(cmdOutput);
      outputFile <- outputs(cmdOutput) if inputs.contains(cmdInput);
      inputFile <- inputs(cmdInput) if outputFile == inputFile
    ) yield outputFile
  }

50
51
52
  @Output
  private var outputFiles: List[File] = Nil

Peter van 't Hof's avatar
Peter van 't Hof committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
  override def beforeGraph(): Unit = {
    val outputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.outputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val inputs: Map[BiopetCommandLineFunction, Seq[File]] = try {
      commands.map(x => x -> x.inputs).toMap
    } catch {
      case e: NullPointerException => Map()
    }

    val fifoFiles = fifos

68
69
70
    outputFiles :::= outputs.values.toList.flatten.filter(!fifoFiles.contains(_))
    outputFiles = outputFiles.distinct

Peter van 't Hof's avatar
Peter van 't Hof committed
71
    deps :::= inputs.values.toList.flatten.filter(!fifoFiles.contains(_))
Peter van 't Hof's avatar
Peter van 't Hof committed
72
    deps = deps.distinct
73

Peter van 't Hof's avatar
Peter van 't Hof committed
74
75
    _pipesJobs :::= commands
    _pipesJobs = _pipesJobs.distinct
Peter van 't Hof's avatar
Peter van 't Hof committed
76
77

    analysisName = commands.map(_.analysisName).mkString("_")
Peter van 't Hof's avatar
Peter van 't Hof committed
78
79
80
  }

  override def beforeCmd(): Unit = {
Peter van 't Hof's avatar
Peter van 't Hof committed
81
    commands.foreach { cmd =>
Peter van 't Hof's avatar
Peter van 't Hof committed
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
      cmd.beforeGraph()
      cmd.internalBeforeGraph()
      cmd.beforeCmd()
    }
  }

  def cmdLine = {
    val fifosFiles = this.fifos
    fifosFiles.filter(_.exists()).map(required("rm", _)).mkString("\n\n", " \n", " \n\n") +
      fifosFiles.map(required("mkfifo", _)).mkString("\n\n", "\n", "\n\n") +
      commands.map(_.commandLine).mkString("\n\n", " & \n", " & \n\n") +
      BiopetFifoPipe.waitScript +
      fifosFiles.map(required("rm", _)).mkString("\n\n", " \n", " \n\n") +
      BiopetFifoPipe.endScript
  }
97
98
99
100

  override def setResources(): Unit = {
    combineResources(commands)
  }
101
102
103
104
105
106

  override def setupRetry(): Unit = {
    super.setupRetry()
    commands.foreach(_.setupRetry())
    combineResources(commands)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
107
108
109
110
111

  override def freezeFieldValues(): Unit = {
    super.freezeFieldValues()
    commands.foreach(_.qSettings = qSettings)
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
112
113
114
115
116
}

object BiopetFifoPipe {
  val waitScript =
    """
117
118
119
120
121
      |
      |allJobs=`jobs -p`
      |jobs=$allJobs
      |
      |echo [`date`] pids: $jobs
Peter van 't Hof's avatar
Peter van 't Hof committed
122
123
124
      |
      |FAIL="0"
      |
Peter van 't Hof's avatar
Peter van 't Hof committed
125
      |while echo $jobs | grep -e "\w" > /dev/null
Peter van 't Hof's avatar
Peter van 't Hof committed
126
      |do
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
      |	for job in $jobs
      |	do
      |		if ps | grep "$job " | grep -v grep > /dev/null
      |		then
      |		    echo [`date`] $job still running > /dev/null
      |		else
      |		    jobs=`echo $jobs | sed "s/${job}//"`
      |			wait $job || FAIL=$?
      |			if echo $FAIL | grep -ve "^0$" > /dev/null
      |			then
      |			    echo [`date`] $job fails with exitcode: $FAIL
      |				break
      |			fi
      |			echo [`date`] $job done
      |		fi
      |	done
      |	if echo $FAIL | grep -ve "^0$" > /dev/null
      |    then
      |        break
      |    fi
      |	sleep 1
Peter van 't Hof's avatar
Peter van 't Hof committed
148
149
      |done
      |
150
151
152
153
154
155
156
157
      |if echo $FAIL | grep -ve "^0$" > /dev/null
      |then
      |    echo [`date`] kill other pids: $jobs
      |    kill $jobs
      |fi
      |
      |echo [`date`] Done
      |
Peter van 't Hof's avatar
Peter van 't Hof committed
158
159
160
161
162
163
164
165
      |
    """.stripMargin

  val endScript =
    """
      |
      |if [ "$FAIL" == "0" ];
      |then
166
      |echo [`date`] "BiopetFifoPipe Done"
Peter van 't Hof's avatar
Peter van 't Hof committed
167
      |else
168
      |echo [`date`] BiopetFifoPipe "FAIL! ($FAIL)"
Peter van 't Hof's avatar
Peter van 't Hof committed
169
170
171
172
173
174
      |exit $FAIL
      |fi
      |
      |
    """.stripMargin
}