AsyncProcess.scala 5.36 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
package nl.lumc.sasc.biopet.utils.process

/*
The MIT License (MIT)
Copyright (c) 2016 j-keck <jhyphenkeck@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

Peter van 't Hof's avatar
WIP  
Peter van 't Hof committed
20 21
import nl.lumc.sasc.biopet.utils.Logging

22 23 24
import scala.collection.parallel.mutable.ParMap
import scala.concurrent.duration.Duration
import scala.concurrent._
Peter van 't Hof's avatar
Peter van 't Hof committed
25
import scala.sys.process.{ Process, ProcessLogger }
26 27 28 29 30 31 32 33 34 35 36
import scala.util.Try

object Sys extends Sys {
  type ExitValue = Int
  type Stdout = String
  type Stderr = String

  type ExecResult = (ExitValue, Stdout, Stderr)

  trait AsyncExecResult {
    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
37 38
     * @see [[scala.concurrent.Future#map]]
     */
39 40 41
    def map[T](f: ExecResult => T): Future[T]

    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
42 43
     * @see [[scala.concurrent.Future#foreach]]
     */
44 45 46
    def foreach(f: ExecResult => Unit): Unit

    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
47 48
     * @see [[scala.concurrent.Future#onComplete]]
     */
49 50 51
    def onComplete[T](pf: Try[ExecResult] => T): Unit

    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
52 53
     * cancels the running process
     */
54 55 56
    def cancel(): Unit

    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
57 58 59
     * check if the process is still running
     * @return `true` if the process is already completed, `false` otherwise
     */
60 61 62
    def isRunning: Boolean

    /**
Peter van 't Hof's avatar
Peter van 't Hof committed
63 64 65
     * the underlying future
     * @return the future, in which the process runs
     */
66 67 68 69 70 71 72 73 74 75 76
    def get: Future[ExecResult]
  }

  type Cancelable = () => Unit

  case class ExecutionCanceled(msg: String) extends Exception(msg)
}

trait Sys {
  import Sys._

77 78 79 80
  private val cache: ParMap[Seq[String], AsyncExecResult] = ParMap()

  var maxRunningProcesses: Int = 5

81 82 83
  def exec(cmd: String): ExecResult = exec(cmd.split(" "))

  /**
Peter van 't Hof's avatar
Peter van 't Hof committed
84 85 86 87 88
   * executes the cmd and blocks until the command exits.
   *
   * @return {{{(ExitValue, Stdout, Stderr)}}}
   *         <pre>if the executable is unable to start, (-1, "", stderr) are returned</pre>
   */
89 90 91 92 93 94 95 96
  def exec(cmd: Seq[String]): ExecResult = {
    val stdout = new OutputSlurper
    val stderr = new OutputSlurper

    Try {
      val proc = Process(cmd).run(ProcessLogger(stdout.appendLine, stderr.appendLine))
      proc.exitValue()
    }.map((_, stdout.get, stderr.get))
Peter van 't Hof's avatar
Peter van 't Hof committed
97
      .recover {
98 99 100 101 102 103 104
        case t => (-1, "", t.getMessage)
      }.get
  }

  def execAsync(cmd: String)(implicit ec: ExecutionContext): AsyncExecResult = execAsync(cmd.split(" "))(ec)

  /**
Peter van 't Hof's avatar
Peter van 't Hof committed
105 106 107 108 109
   * executes the cmd asynchronous
   * @see scala.concurrent.Future.map
   *
   * @return [[AsyncExecResult]]
   */
110
  def execAsync(cmd: Seq[String])(implicit ec: ExecutionContext): AsyncExecResult = {
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
    while (cache.size >= maxRunningProcesses) {
      for ((cmd, c) <- cache.toList) {
        val results = Option(c)
        if (!results.map(_.isRunning).getOrElse(true)) try {
          cache -= cmd
        } catch {
          case e: NullPointerException =>
        }
        else try {
          results.foreach(x => Await.ready(x.get, Duration.fromNanos(100000)))
        } catch {
          case e: TimeoutException =>
        }
      }
    }
    val results = new AsyncExecResult {
127 128 129 130 131 132 133 134 135 136 137 138 139 140
      val (fut, cancelFut) = runAsync(cmd)

      override def map[T](f: ExecResult => T): Future[T] = fut.map(f)

      override def foreach(f: ExecResult => Unit): Unit = fut.foreach(f)

      override def onComplete[T](pf: Try[ExecResult] => T): Unit = fut.onComplete(pf)

      override def cancel(): Unit = cancelFut()

      override def isRunning: Boolean = !fut.isCompleted

      override def get: Future[ExecResult] = fut
    }
141 142
    cache += cmd -> results
    results
143 144 145 146 147 148 149 150 151 152 153 154 155 156
  }

  // helper for 'execAsync' - runs the given cmd asynchronous.
  // returns a tuple with: (the running process in a future, function to cancel the running process)
  private def runAsync(cmd: Seq[String])(implicit ec: ExecutionContext): (Future[ExecResult], Cancelable) = {
    val p = Promise[ExecResult]

    val stdout = new OutputSlurper
    val stderr = new OutputSlurper

    // start the process
    val proc = Process(cmd).run(ProcessLogger(stdout.appendLine, stderr.appendLine))
    p.tryCompleteWith(Future(proc.exitValue).map(c => (c, stdout.get, stderr.get)))

Peter van 't Hof's avatar
WIP  
Peter van 't Hof committed
157 158 159 160 161 162
    val cancel = () => {
      p.tryFailure {
        Logging.logger.error("stdout: " + stdout.get)
        Logging.logger.error("stderr: " + stderr.get)
        new ExecutionCanceled(s"Process: '${cmd.mkString(" ")}' canceled")
      }
163 164 165
      proc.destroy()
    }

Peter van 't Hof's avatar
WIP  
Peter van 't Hof committed
166
    (p.future, cancel)
167 168 169 170 171 172 173 174 175 176 177
  }

  class OutputSlurper {
    private val sb = new StringBuilder

    def append(s: String): Unit = sb.append(s)

    def appendLine(s: String): Unit = append(s + "\n")

    def get: String = sb.toString
  }
Peter van 't Hof's avatar
Peter van 't Hof committed
178
}