Merge pull request #305 from biowdl/BIOWDL-596

Use gebibytes instead of gigabytes

CHANGELOG.md

@@ -9,6 +9,11 @@ that users understand how the changes affect the new version.
 -->
 version 5.1.0-dev
 ---------------------------
++ Move all memory notation to `KiB`, `MiB` and `GiB` from `K`, `M` and `G` 
+  previously. The WDL spec clearly distuingishes between SI and binary 
+  notations. Since Java always takes `K`, `M` and `G` to mean `KiB`, `MiB` and 
+  `GiB` this means java tasks such as GATK, FastQC and Picard will always 
+  receive enough memory now.  
 + Purple's `somaticRainfallPlot` output is now optional and included in
   the `plots` output as well.
 + Bedtools coverage's timeMinutes now defaults to `320`.

CPAT.wdl

@@ -34,8 +34,8 @@ task CPAT {
         Array[String]? startCodons
         Array[String]? stopCodons
 
-        String memory = "4G"
-        Int timeMinutes = 10 + ceil(size(gene, "G") * 30)
+        String memory = "4GiB"
+        Int timeMinutes = 10 + ceil(size(gene, "GiB") * 30)
         String dockerImage = "quay.io/biocontainers/cpat:3.0.4--py39hcbe4a3b_0"
     }
 

bam2fastx.wdl

@@ -30,7 +30,7 @@ task Bam2Fasta {
 
         String? seqIdPrefix
 
-        String memory = "2G"
+        String memory = "2GiB"
         Int timeMinutes = 15
         String dockerImage = "quay.io/biocontainers/bam2fastx:1.3.1--hf05d43a_1"
     }
@@ -98,7 +98,7 @@ task Bam2Fastq {
 
         String? seqIdPrefix
 
-        String memory = "2G"
+        String memory = "2GiB"
         Int timeMinutes = 15
         String dockerImage = "quay.io/biocontainers/bam2fastx:1.3.1--hf05d43a_1"
     }

bcftools.wdl

@@ -47,7 +47,7 @@ task Annotate {
         File? samplesFile
 
         Int threads = 0
-        String memory = "4G"
+        String memory = "4GiB"
         Int timeMinutes = 60 + ceil(size(inputFile, "G"))
         String dockerImage = "quay.io/biocontainers/bcftools:1.10.2--h4f4756c_2"
     }
@@ -138,7 +138,7 @@ task Filter {
         String? softFilter
         String outputPath = "./filtered.vcf.gz"
 
-        String memory = "256M"
+        String memory = "256MiB"
         Int timeMinutes = 1 + ceil(size(vcf, "G"))
         String dockerImage = "quay.io/biocontainers/bcftools:1.10.2--h4f4756c_2"
     }
@@ -186,7 +186,7 @@ task Sort {
         String outputPath = "output.vcf.gz"
         String tmpDir = "./sorting-tmp"
 
-        String memory = "5G"
+        String memory = "5GiB"
         Int timeMinutes = 1 + ceil(size(inputFile, "G")) * 5
         String dockerImage = "quay.io/biocontainers/bcftools:1.10.2--h4f4756c_2"
     }
@@ -261,7 +261,7 @@ task Stats {
         String? userTsTv
 
         Int threads = 0
-        String memory = "256M"
+        String memory = "256MiB"
         Int timeMinutes = 1 + 2* ceil(size(select_all([inputVcf, compareVcf]), "G")) # TODO: Estimate, 2 minutes per GB, refine later.
         String dockerImage = "quay.io/biocontainers/bcftools:1.10.2--h4f4756c_2"
     }
@@ -350,7 +350,7 @@ task View {
         String? exclude
         String? include
 
-        String memory = "256M"
+        String memory = "256MiB"
         Int timeMinutes = 1 + ceil(size(inputFile, "G"))
         String dockerImage = "quay.io/biocontainers/bcftools:1.10.2--h4f4756c_2"
     }

bedtools.wdl

@@ -26,7 +26,7 @@ task Complement {
         File inputBed
         String outputBed = basename(inputBed, "\.bed") + ".complement.bed"
 
-        String memory = "~{512 + ceil(size([inputBed, faidx], "M"))}M"
+        String memory = "~{512 + ceil(size([inputBed, faidx], "MiB"))}MiB"
         Int timeMinutes = 1 + ceil(size([inputBed, faidx], "G"))
         String dockerImage = "quay.io/biocontainers/bedtools:2.23.0--hdbcaa40_3"
     }
@@ -75,7 +75,7 @@ task Coverage {
         File? bIndex
         String outputPath = "./coverage.tsv"
 
-        String memory = "8G"
+        String memory = "8GiB"
         Int timeMinutes = 320
         String dockerImage = "quay.io/biocontainers/bedtools:2.30.0--h7d7f7ad_2"
     }
@@ -120,7 +120,7 @@ task Merge {
         File inputBed
         String outputBed = "merged.bed"
 
-        String memory = "~{512 + ceil(size(inputBed, "M"))}M"
+        String memory = "~{512 + ceil(size(inputBed, "MiB"))}MiB"
         Int timeMinutes = 1 + ceil(size(inputBed, "G"))
         String dockerImage = "quay.io/biocontainers/bedtools:2.23.0--hdbcaa40_3"
     }
@@ -159,7 +159,7 @@ task MergeBedFiles {
         Array[File]+ bedFiles
         String outputBed = "merged.bed"
 
-        String memory = "~{512 + ceil(size(bedFiles, "M"))}M"
+        String memory = "~{512 + ceil(size(bedFiles, "MiB"))}MiB"
         Int timeMinutes = 1 + ceil(size(bedFiles, "G"))
         String dockerImage = "quay.io/biocontainers/bedtools:2.23.0--hdbcaa40_3"
     }
@@ -207,8 +207,8 @@ task Sort {
         File? genome
         File? faidx
 
-        String memory = "~{512 + ceil(size(inputBed, "M"))}M"
-        Int timeMinutes = 1 + ceil(size(inputBed, "G"))
+        String memory = "~{512 + ceil(size(inputBed, "MiB"))}MiB"
+        Int timeMinutes = 1 + ceil(size(inputBed, "GiB"))
         String dockerImage = "quay.io/biocontainers/bedtools:2.23.0--hdbcaa40_3"
     }
 
@@ -267,8 +267,8 @@ task Intersect {
 
         File? faidx # Giving a faidx file will set the sorted option.
 
-        String memory = "~{512 + ceil(size([regionsA, regionsB], "M"))}M"
-        Int timeMinutes = 1 + ceil(size([regionsA, regionsB], "G"))
+        String memory = "~{512 + ceil(size([regionsA, regionsB], "MiB"))}MiB"
+        Int timeMinutes = 1 + ceil(size([regionsA, regionsB], "GiB"))
         String dockerImage = "quay.io/biocontainers/bedtools:2.23.0--hdbcaa40_3"
     }
 

biowdl.wdl

@@ -32,7 +32,7 @@ task InputConverter {
         Boolean checkFileMd5sums=false
         Boolean old=false
 
-        String memory = "128M"
+        String memory = "128MiB"
         Int timeMinutes = 1
         String dockerImage = "quay.io/biocontainers/biowdl-input-converter:0.3.0--pyhdfd78af_0"
     }

bowtie.wdl

@@ -37,7 +37,7 @@ task Bowtie {
 
         String picardXmx = "4G"
         Int threads = 1
-        String memory = "~{5 + ceil(size(indexFiles, "G"))}G"
+        String memory = "~{5 + ceil(size(indexFiles, "GiB"))}GiB"
         Int timeMinutes = 1 + ceil(size(flatten([readsUpstream, readsDownstream]), "G") * 300 / threads)
         # Image contains bowtie=1.2.2 and picard=2.9.2
         String dockerImage = "quay.io/biocontainers/mulled-v2-bfe71839265127576d3cd749c056e7b168308d56:1d8bec77b352cdcf3e9ff3d20af238b33ed96eae-0"

bwa-mem2.wdl

@@ -36,7 +36,7 @@ task Mem {
         
         Int threads = 4
         Int? memoryGb
-        Int timeMinutes = 1 + ceil(size([read1, read2], "G") * 220 / threads)
+        Int timeMinutes = 1 + ceil(size([read1, read2], "GiB") * 220 / threads)
         # Contains bwa-mem2 2.0 bwakit 0.7.17.dev1 and samtools 1.10.
         String dockerImage = "quay.io/biocontainers/mulled-v2-6a15c99309c82b345497d24489bee67bbb76c2f6:1c9c3227b9bf825a8dc9726a25701aa23c0b1f12-0"
     }
@@ -84,7 +84,7 @@ task Mem {
         # One extra thread for bwa-postalt + samtools is not needed.
         # These only use 5-10% of compute power and not always simultaneously.
         cpu: threads
-        memory: "~{select_first([memoryGb, estimatedMemoryGb])}G"
+        memory: "~{select_first([memoryGb, estimatedMemoryGb])}GiB"
         time_minutes: timeMinutes
         docker: dockerImage
     }

bwa.wdl

@@ -37,7 +37,7 @@ task Mem {
 
         Int threads = 4
         Int? memoryGb
-        Int timeMinutes = 10 + ceil(size([read1, read2], "G") * 300 / threads)
+        Int timeMinutes = 10 + ceil(size([read1, read2], "GiB") * 300 / threads)
         # Contains bwa 0.7.17 bwakit 0.7.17.dev1 and samtools 1.10.
         String dockerImage = "quay.io/biocontainers/mulled-v2-ad317f19f5881324e963f6a6d464d696a2825ab6:c59b7a73c87a9fe81737d5d628e10a3b5807f453-0"
     }
@@ -48,7 +48,7 @@ task Mem {
     Int estimatedSortThreads = if threads == 1 then 1 else 1 + ceil(threads / 4.0)
     Int totalSortThreads = select_first([sortThreads, estimatedSortThreads])
     # BWA needs slightly more memory than the size of the index files (~10%). Add a margin for safety here.
-    Int estimatedMemoryGb = 10 + ceil(size(bwaIndex.indexFiles, "G") * 2) + sortMemoryPerThreadGb * totalSortThreads
+    Int estimatedMemoryGb = 10 + ceil(size(bwaIndex.indexFiles, "GiB") * 2) + sortMemoryPerThreadGb * totalSortThreads
     
     # The bwa postalt script is out commented as soon as usePostalt = false.
     # This hack was tested with bash, dash and ash. It seems that comments in between pipes work for all of them.
@@ -81,7 +81,7 @@ task Mem {
         # One extra thread for bwa-postalt + samtools is not needed.
         # These only use 5-10% of compute power and not always simultaneously.
         cpu: threads
-        memory: "~{select_first([memoryGb, estimatedMemoryGb])}G"
+        memory: "~{select_first([memoryGb, estimatedMemoryGb])}GiB"
         time_minutes: timeMinutes
         docker: dockerImage
     }

ccs.wdl

@@ -42,7 +42,7 @@ task CCS {
         String? chunkString
 
         Int threads = 2
-        String memory = "4G"
+        String memory = "4GiB"
         Int timeMinutes = 1440
         String dockerImage = "quay.io/biocontainers/pbccs:6.0.0--h9ee0642_2"
     }

centrifuge.wdl

@@ -36,7 +36,7 @@ task Build {
         File? sizeTable
 
         Int threads = 5
-        String memory = "20G"
+        String memory = "20GiB"
         Int timeMinutes = 2880
         String dockerImage = "quay.io/biocontainers/centrifuge:1.0.4_beta--he513fc3_5"
     }
@@ -109,7 +109,7 @@ task Classify {
         String? excludeTaxIDs
 
         Int threads = 4
-        String memory = "16G"
+        String memory = "16GiB"
         Int timeMinutes = 2880
         String dockerImage = "quay.io/biocontainers/centrifuge:1.0.4_beta--he513fc3_5"
     }
@@ -186,7 +186,7 @@ task Inspect {
 
         Int? across
 
-        String memory = "4G"
+        String memory = "4GiB"
         Int timeMinutes = 1
         String dockerImage = "quay.io/biocontainers/centrifuge:1.0.4_beta--he513fc3_5"
     }
@@ -245,7 +245,7 @@ task KReport {
         Int? minimumScore
         Int? minimumLength
 
-        String memory = "4G"
+        String memory = "4GiB"
         Int timeMinutes = 10
         String dockerImage = "quay.io/biocontainers/centrifuge:1.0.4_beta--he513fc3_5"
     }
@@ -303,7 +303,7 @@ task KTimportTaxonomy {
         File inputFile
         String outputPrefix
 
-        String memory = "4G"
+        String memory = "4GiB"
         Int timeMinutes = 1
         String dockerImage = "biocontainers/krona:v2.7.1_cv1"
     }

chunked-scatter.wdl

@@ -30,7 +30,7 @@ task ChunkedScatter {
         Int? overlap
         Int? minimumBasesPerFile
 
-        String memory = "256M"
+        String memory = "256MiB"
         Int timeMinutes = 2
         String dockerImage = "quay.io/biocontainers/chunked-scatter:1.0.0--py_0"
     }
@@ -84,7 +84,7 @@ task ScatterRegions {
 
         Int? scatterSize
 
-        String memory = "256M"
+        String memory = "256MiB"
         Int timeMinutes = 2
         String dockerImage = "quay.io/biocontainers/chunked-scatter:1.0.0--py_0"
     }

clever.wdl

@@ -34,7 +34,7 @@ task Mateclever {
         Int maxOffset = 150
 
         Int threads = 10
-        String memory = "15G"
+        String memory = "15GiB"
         Int timeMinutes = 600
         String dockerImage = "quay.io/biocontainers/clever-toolkit:2.4--py36hcfe0e84_6"
     }
@@ -94,7 +94,7 @@ task Prediction {
         String outputPath = "./clever"
 
         Int threads = 10
-        String memory = "55G"
+        String memory = "55GiB"
         Int timeMinutes = 480
         String dockerImage = "quay.io/biocontainers/clever-toolkit:2.4--py36hcfe0e84_6"
     }

collect-columns.wdl

@@ -62,7 +62,7 @@ task CollectColumns {
     }
 
     runtime {
-        memory: "~{memoryGb}G"
+        memory: "~{memoryGb}GiB"
         time_minutes: timeMinutes
         docker: dockerImage
     }

common.wdl

@@ -25,7 +25,7 @@ task AppendToStringArray {
         Array[String] array
         String string
         
-        String memory = "1G"
+        String memory = "1GiB"
     }
 
     command {
@@ -51,7 +51,7 @@ task CheckFileMD5 {
         # By default cromwell expects /bin/bash to be present in the container.
         # The 'bash' container does not fill this requirement. (It is in /usr/local/bin/bash)
         # Use a stable version of debian:stretch-slim for this. (Smaller than ubuntu)
-        String memory = "1G"
+        String memory = "1GiB"
         String dockerImage = "debian@sha256:f05c05a218b7a4a5fe979045b1c8e2a9ec3524e5611ebfdd0ef5b8040f9008fa"
     }
 
@@ -75,7 +75,7 @@ task ConcatenateTextFiles {
         Boolean unzip = false
         Boolean zip = false
 
-        String memory = "1G"
+        String memory = "1GiB"
     }
 
     # When input and output is both compressed decompression is not needed.
@@ -104,7 +104,7 @@ task Copy {
         Boolean recursive = false
 
         # Version not that important as long as it is stable.
-        String memory = "1G"
+        String memory = "1GiB"
         String dockerImage = "debian@sha256:f05c05a218b7a4a5fe979045b1c8e2a9ec3524e5611ebfdd0ef5b8040f9008fa"
     }
 
@@ -132,7 +132,7 @@ task CreateLink {
         String inputFile
         String outputPath
 
-        String memory = "1G"
+        String memory = "1GiB"
     }
 
     command {
@@ -170,7 +170,7 @@ task GetSamplePositionInArray {
 
     runtime {
         # 4 gigs of memory to be able to build the docker image in singularity.
-        memory: "4G"
+        memory: "4GiB"
         docker: dockerImage
         timeMinutes: 5
     }
@@ -190,7 +190,7 @@ task MapMd5 {
     input {
         Map[String,String] map
 
-        String memory = "1G"
+        String memory = "1GiB"
         String dockerImage = "debian@sha256:f05c05a218b7a4a5fe979045b1c8e2a9ec3524e5611ebfdd0ef5b8040f9008fa"
     }
 
@@ -214,7 +214,7 @@ task StringArrayMd5 {
     input {
         Array[String] stringArray
 
-        String memory = "1G"
+        String memory = "1GiB"
         String dockerImage = "debian@sha256:f05c05a218b7a4a5fe979045b1c8e2a9ec3524e5611ebfdd0ef5b8040f9008fa"
     }
 
@@ -238,7 +238,7 @@ task TextToFile {
         String text
         String outputFile = "out.txt"
 
-        String memory = "1G"
+        String memory = "1GiB"
         Int timeMinutes = 1
         String dockerImage = "debian@sha256:f05c05a218b7a4a5fe979045b1c8e2a9ec3524e5611ebfdd0ef5b8040f9008fa"
     }
@@ -274,7 +274,7 @@ task YamlToJson {
         File yaml
         String outputJson = basename(yaml, "\.ya?ml$") + ".json"
 
-        String  memory = "128M"
+        String  memory = "128MiB"
         Int timeMinutes = 1
         # biowdl-input-converter has python and pyyaml.
         String dockerImage = "quay.io/biocontainers/biowdl-input-converter:0.3.0--pyhdfd78af_0"

cutadapt.wdl

@@ -83,7 +83,7 @@ task Cutadapt {
         Boolean? noZeroCap
 
         Int cores = 4
-        String memory = "5G"
+        String memory = "5GiB"
         Int timeMinutes = 1 + ceil(size([read1, read2], "G")  * 12.0 / cores)
         String dockerImage = "quay.io/biocontainers/cutadapt:2.10--py37hf01694f_1"
     }

deconstructsigs.wdl

@@ -27,7 +27,7 @@ task DeconstructSigs {
         String outputPath = "./signatures.rds"
 
         Int timeMinutes = 15
-        String memory = "4G"
+        String memory = "4GiB"
         String dockerImage = "quay.io/biocontainers/r-deconstructsigs:1.9.0--r41hdfd78af_1"
     }
 

deepvariant.wdl

@@ -37,7 +37,7 @@ task RunDeepVariant {
         String? sampleName
         Boolean? VCFStatsReport = true
 
-        String memory = "3G"
+        String memory = "3GiB"
         Int timeMinutes = 5000
         String dockerImage = "google/deepvariant:1.0.0"
     }

delly.wdl

@@ -28,7 +28,7 @@ task CallSV {
         File referenceFastaFai
         String outputPath = "./delly/delly.bcf"
 
-        String memory = "15G"
+        String memory = "15GiB"
         Int timeMinutes = 300
         String dockerImage = "quay.io/biocontainers/delly:0.8.1--h4037b6b_1"
     }

duphold.wdl

@@ -30,7 +30,7 @@ task Duphold {
         String sample
         String outputPath = "./duphold.vcf"
 
-        String memory = "15G"
+        String memory = "15GiB"
         Int timeMinutes = 1440
         String dockerImage = "quay.io/biocontainers/duphold:0.2.1--h516909a_1"
     }