Commit 4f9286e4 authored by Hoogenboom, Jerry's avatar Hoogenboom, Jerry
Browse files

Various fixes and improvements

Fixed:
* Fixed a crash in BGMerge.
* Fixed bug in BGCorrect that resulted in incorrect values in the
  *_add and *_corrected columns (yes, you, 8685a304).
* Fixed a glitch in BGCorrect that prevented it from ever writing
  corrected_bgestimate in the correction_flags column.

Improved:
* BGEstimate will now include the sample tag in the error messages for
  missing alleles and alleles with 0 reads.
* Strand bias lines in Samplevis are now clamped to the 0-100% range.
  BGCorrect may cause forward read percentages outside this range.

Visualisations:
* Updated Vega to version 2.4.2.
* Fixed drag-'n-drop behaviour for HTML visualisations in Internet
  Explorer and Firefox.
* Fixed the Save Image link when viewing HTML visualisations in
  Internet Explorer 10 and above.
* Added http-equiv="X-UA-Compatible" content="IE=edge" meta-tag to all
  visualisations to prevent Internet Explorer from entering quirks mode.
* Samplevis:
  * Fixed glitch that would sometimes cause a second horizontal scroll
    bar to appear.
  * Graphs now render much more quickly when 'Split markers' is on, and
    Chrome no longer crashes on large sample files with this option set.
parent a3e610e8
......@@ -117,8 +117,8 @@ def match_profile(column_names, data, profile, convert_to_raw, library,
np.fill_diagonal(P2, 0)
forward_noise = A * P1
reverse_noise = A * P2
forward_add = np.multiply(A, P1.sum(1))
reverse_add = np.multiply(A, P2.sum(1))
forward_add = np.multiply(A, P1.sum(1).T)
reverse_add = np.multiply(A, P2.sum(1).T)
# Round values to 3 decimal positions.
forward_noise.round(3, forward_noise);
......@@ -149,7 +149,7 @@ def match_profile(column_names, data, profile, convert_to_raw, library,
line[colid_total_corrected] += line[colid_total_add]
if "bgestimate" in profile["tool"][i]:
line[colid_correction_flags] = "corrected_bgestimate"
if "bghomstats" in profile["tool"][i]:
elif "bghomstats" in profile["tool"][i]:
line[colid_correction_flags] = "corrected_bghomstats"
elif "bgpredict" in profile["tool"][i]:
line[colid_correction_flags] = "corrected_bgpredict"
......
......@@ -287,7 +287,7 @@ def ensure_min_samples(allelelist, min_samples):
#ensure_min_samples
def add_sample_data(data, sample_data, sample_alleles, min_pct, min_abs):
def add_sample_data(data, sample_data, sample_alleles, min_pct, min_abs, tag):
# Make sure the true alleles of this sample are added to data.
# Also compute the allele-specific inclusion thresholds for noise.
thresholds = {}
......@@ -311,10 +311,12 @@ def add_sample_data(data, sample_data, sample_alleles, min_pct, min_abs):
for allele in sample_alleles[marker]:
if (marker, allele) not in sample_data:
raise ValueError(
"Missing allele %s of marker %s!" % (allele, marker))
"Missing allele %s of marker %s in sample %s!" %
(allele, marker, tag))
elif 0 in sample_data[marker, allele]:
raise ValueError(
"Allele %s of marker %s has 0 reads!" % (allele, marker))
"Allele %s of marker %s has 0 reads on one strand in "
"sample %s!" % (allele, marker, tag))
try:
i = p["alleles"].index(allele)
except ValueError:
......@@ -420,7 +422,7 @@ def generate_profiles(samples_in, outfile, reportfile, allelefile,
data = {}
for tag in sample_data.keys():
add_sample_data(data, sample_data[tag], allelelist[tag], min_pct,
min_abs)
min_abs, tag)
del sample_data[tag]
# Filter insignificant background products.
......
......@@ -43,7 +43,7 @@ def merge_profiles(infiles, outfile, seqformat, library):
if key not in amounts[marker]:
this_amounts = (profiles[marker]["forward"][i][j],
profiles[marker]["reverse"][i][j],
profiles[marker]["tools"][i][j])
profiles[marker]["tool"][i][j])
if sum(this_amounts[:2]):
amounts[marker][key] = this_amounts
......
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="UTF-8">
<title>Allele Visualisation - FDSTools</title>
<!-- BEGIN_LIBRARIES -->
......@@ -130,7 +131,7 @@
function parse(){
var this_stamp = ++stamp;
vg.parse.spec(graph_spec, function(chart){
// Cancel rendering if a new parse() call was made.
//Cancel rendering if a new parse() call was made.
if(this_stamp != stamp)
return;
......@@ -181,9 +182,11 @@
}
//Load the data (input is a fileList object; only the first file is loaded).
var currentlyLoadedFile = "no/file/loaded";
function loadDataset(fileList){
if(!graph_spec || !fileList || !fileList.length)
if(!graph_spec || !fileList || !fileList.length || fileList[0].name == currentlyLoadedFile)
return;
currentlyLoadedFile = fileList[0].name;
var reader = new FileReader();
reader.onload = function(e){
if(fileList && fileList.length && fileList[0].name)
......@@ -201,11 +204,24 @@
function saveImage(){
var link = document.getElementById("saveLink");
var imageType = document.getElementById("renderSVG").checked? "svg": "png";
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
if(window.navigator.msSaveOrOpenBlob){
//Internet Explorer has its own ways of doing things.
var b;
if(imageType == "svg")
b = new Blob([graph._el.innerHTML], {type: "image/svg+xml;charset=utf-8"});
else
b = graph._el.firstChild.msToBlob();
window.navigator.msSaveOrOpenBlob(b, fileName + "." + imageType);
if(b.msClose)
b.msClose();
}
else{
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
}
return false;
}
......@@ -225,7 +241,13 @@
document.addEventListener('drop', function(evt){
evt.stopPropagation();
evt.preventDefault();
//Try to clear the currently displayed file, then try to set
//the drag-'n-dropped file as the selected one on the file input.
//Both actions are not supported in all major browsers.
document.getElementById("fileselect").value = "";
document.getElementById("fileselect").files = evt.dataTransfer.files;
loadDataset(evt.dataTransfer.files);
}, false);
}
......
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="UTF-8">
<title>Background Noise Visualisation - FDSTools</title>
<!-- BEGIN_LIBRARIES -->
......@@ -133,7 +134,7 @@
function parse(){
var this_stamp = ++stamp;
vg.parse.spec(graph_spec, function(chart){
// Cancel rendering if a new parse() call was made.
//Cancel rendering if a new parse() call was made.
if(this_stamp != stamp)
return;
......@@ -204,9 +205,11 @@
}
//Load the data (input is a fileList object; only the first file is loaded).
var currentlyLoadedFile = "no/file/loaded";
function loadDataset(fileList){
if(!graph_spec || !fileList || !fileList.length)
if(!graph_spec || !fileList || !fileList.length || fileList[0].name == currentlyLoadedFile)
return;
currentlyLoadedFile = fileList[0].name;
var reader = new FileReader();
reader.onload = function(e){
if(fileList && fileList.length && fileList[0].name)
......@@ -224,11 +227,24 @@
function saveImage(){
var link = document.getElementById("saveLink");
var imageType = document.getElementById("renderSVG").checked? "svg": "png";
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
if(window.navigator.msSaveOrOpenBlob){
//Internet Explorer has its own ways of doing things.
var b;
if(imageType == "svg")
b = new Blob([graph._el.innerHTML], {type: "image/svg+xml;charset=utf-8"});
else
b = graph._el.firstChild.msToBlob();
window.navigator.msSaveOrOpenBlob(b, fileName + "." + imageType);
if(b.msClose)
b.msClose();
}
else{
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
}
return false;
}
......@@ -248,7 +264,13 @@
document.addEventListener('drop', function(evt){
evt.stopPropagation();
evt.preventDefault();
//Try to clear the currently displayed file, then try to set
//the drag-'n-dropped file as the selected one on the file input.
//Both actions are not supported in all major browsers.
document.getElementById("fileselect").value = "";
document.getElementById("fileselect").files = evt.dataTransfer.files;
loadDataset(evt.dataTransfer.files);
}, false);
}
......
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="UTF-8">
<title>Background Noise Profile Visualisation - FDSTools</title>
<!-- BEGIN_LIBRARIES -->
......@@ -129,7 +130,7 @@
function parse(){
var this_stamp = ++stamp;
vg.parse.spec(graph_spec, function(chart){
// Cancel rendering if a new parse() call was made.
//Cancel rendering if a new parse() call was made.
if(this_stamp != stamp)
return;
......@@ -201,9 +202,11 @@
}
//Load the data (input is a fileList object; only the first file is loaded).
var currentlyLoadedFile = "no/file/loaded";
function loadDataset(fileList){
if(!graph_spec || !fileList || !fileList.length)
if(!graph_spec || !fileList || !fileList.length || fileList[0].name == currentlyLoadedFile)
return;
currentlyLoadedFile = fileList[0].name;
var reader = new FileReader();
reader.onload = function(e){
if(fileList && fileList.length && fileList[0].name)
......@@ -221,11 +224,24 @@
function saveImage(){
var link = document.getElementById("saveLink");
var imageType = document.getElementById("renderSVG").checked? "svg": "png";
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
if(window.navigator.msSaveOrOpenBlob){
//Internet Explorer has its own ways of doing things.
var b;
if(imageType == "svg")
b = new Blob([graph._el.innerHTML], {type: "image/svg+xml;charset=utf-8"});
else
b = graph._el.firstChild.msToBlob();
window.navigator.msSaveOrOpenBlob(b, fileName + "." + imageType);
if(b.msClose)
b.msClose();
}
else{
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
}
return false;
}
......@@ -245,7 +261,13 @@
document.addEventListener('drop', function(evt){
evt.stopPropagation();
evt.preventDefault();
//Try to clear the currently displayed file, then try to set
//the drag-'n-dropped file as the selected one on the file input.
//Both actions are not supported in all major browsers.
document.getElementById("fileselect").value = "";
document.getElementById("fileselect").files = evt.dataTransfer.files;
loadDataset(evt.dataTransfer.files);
}, false);
}
......
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="UTF-8">
<title>Sample Visualisation - FDSTools</title>
<!-- BEGIN_LIBRARIES -->
......@@ -60,7 +61,6 @@
text-align: right;
}
div#vis {
overflow: auto;
direction: rtl;
display: inline-block;
}
......@@ -335,6 +335,8 @@
</div>
<script type="text/javascript">
var graphs = false;
var data_values = {};
var data_format = {};
var fileName = "graph";
var stamp = 0;
var userSelected = [];
......@@ -355,13 +357,13 @@
graphs = [];
removeChildren(visdiv);
// Get a list of marker names and render a graph for each of them.
//Get a list of marker names and render a graph for each of them.
var splitMarkers = document.getElementById("splitmarkers").checked;
var markerFilter = document.getElementById("markerFilter").value;
var markerNames = [];
if(splitMarkers){
var patt = new RegExp('(?:' + markerFilter.trim().replace(/(^| )=(.*?)(?= |$)/g, '$1^$2$$').replace(/ +/g, ')|(?:') + ')');
vg.util.read(graph_spec.data[0].values, {type: 'tsv'}).forEach(function(datum){
data_values.forEach(function(datum){
if(patt.test(datum.name))
markerNames.push("=" + datum.name);
});
......@@ -375,18 +377,22 @@
else
return;
// Render graph for this marker.
//Set up graph spec for this marker.
for(i in graph_spec.signals)
if(graph_spec.signals[i].name == "filter_marker")
graph_spec.signals[i].init = marker;
var patt = new RegExp('(?:' + marker.replace(/(^| )=(.*?)(?= |$)/g, '$1^$2$$').replace(/ +/g, ')|(?:') + ')');
graph_spec.data[0].values = data_values.filter(function(datum){return patt.test(datum.name)});
//Render graph for this marker.
vg.parse.spec(graph_spec, function(chart){
// Cancel rendering if a new parse() call was made.
//Cancel rendering if a new parse() call was made.
if(this_stamp != stamp)
return;
var i = graphs.length;
// Add new visualisation and table divs.
//Add new visualisation and table divs.
if(i > 0)
visdiv.appendChild(document.createElement("hr"));
var newvis = document.createElement("div");
......@@ -431,8 +437,8 @@
graphs.push(graph);
document.getElementById("saveLink").style.display = (splitMarkers? "none" : "inline");
// Restore selected and deselected alleles. Deselected alleles
// are the ones that appear in selectedAlleles twice.
//Restore selected and deselected alleles. Deselected alleles
//are the ones that appear in selectedAlleles twice.
var graphAlleles = graph.data("annotated").values();
selectedAlleles = selectedAlleles.filter(function(sa){
return graphAlleles.every(function(datum){
......@@ -453,10 +459,10 @@
});
});
// Automatically select alleles using set thresholds.
//Automatically select alleles using set thresholds.
autoSelectAlleles(graph);
// Scroll to the right; the graph is more interesting than the long labels.
//Scroll to the right; the graph is more interesting than the long labels.
scrolldiv.scrollLeft = scrolldiv.scrollWidth;
});
});
......@@ -480,13 +486,13 @@
datum.total_added >= minNa &&
datum.pct_of_max >= minPa &&
datum.pct_of_sum >= minTa &&
((datum.total_added/datum.total*100-100) >= minCa || // TODO: Docs!!
((datum.total_added/datum.total*100-100) >= minCa || //TODO: Docs!!
datum.total_add >= minAa) &&
Math.min(datum.forward_added, datum.reverse_added) >= minOa){
if(autoSelected.indexOf(datum) == -1)
autoSelected.push(datum);
if(selectedAlleles.indexOf(datum) == -1 && userDeselected.indexOf(datum) == -1){
// Meets criteria, not selected, and user has not deselected. Autoselect.
//Meets criteria, not selected, and user has not deselected. Autoselect.
isAutoselecting = true;
graph.signal("clickedAllele", datum).update();
isAutoselecting = false;
......@@ -497,7 +503,7 @@
if(pos >= 0)
autoSelected.splice(pos, 1);
if(selectedAlleles.indexOf(datum) >= 0 && userSelected.indexOf(datum) == -1){
// Does not meet criteria but was autoselected previously. Autodeselect.
//Does not meet criteria but was autoselected previously. Autodeselect.
isAutoselecting = true;
graph.signal("clickedAllele", datum).update();
isAutoselecting = false;
......@@ -509,7 +515,7 @@
}
function updateViewBox(svg){
// Setting the viewBox enables shrink-to-fit behaviour when printing.
//Setting the viewBox enables shrink-to-fit behaviour when printing.
svg.setAttribute("viewBox",
"0 0 " + svg.getAttribute("width") + " " + svg.getAttribute("height"));
}
......@@ -590,9 +596,11 @@
}
//Load the data (input is a fileList object; only the first file is loaded).
var currentlyLoadedFile = "no/file/loaded";
function loadDataset(fileList){
if(!graph_spec || !fileList || !fileList.length)
if(!graph_spec || !fileList || !fileList.length || fileList[0].name == currentlyLoadedFile)
return;
currentlyLoadedFile = fileList[0].name;
var reader = new FileReader();
reader.onload = function(e){
if(fileList && fileList.length && fileList[0].name)
......@@ -600,7 +608,7 @@
else
fileName = "graph";
document.title = fileName + " - Sample Visualisation - FDSTools";
graph_spec.data[0].values = reader.result;
data_values = vg.util.read(reader.result, data_format);
parse();
};
reader.readAsText(fileList[0]);
......@@ -610,11 +618,24 @@
function saveImage(){
var link = document.getElementById("saveLink");
var imageType = document.getElementById("renderSVG").checked? "svg": "png";
link.setAttribute("href", graphs[0].toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
if(window.navigator.msSaveOrOpenBlob){
//Internet Explorer has its own ways of doing things.
var b;
if(imageType == "svg")
b = new Blob([graphs[0]._el.innerHTML], {type: "image/svg+xml;charset=utf-8"});
else
b = graphs[0]._el.firstChild.msToBlob();
window.navigator.msSaveOrOpenBlob(b, fileName + "." + imageType);
if(b.msClose)
b.msClose();
}
else{
link.setAttribute("href", graphs[0].toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
}
return false;
}
......@@ -710,10 +731,23 @@
document.addEventListener('drop', function(evt){
evt.stopPropagation();
evt.preventDefault();
//Try to clear the currently displayed file, then try to set
//the drag-'n-dropped file as the selected one on the file input.
//Both actions are not supported in all major browsers.
document.getElementById("fileselect").value = "";
document.getElementById("fileselect").files = evt.dataTransfer.files;
loadDataset(evt.dataTransfer.files);
}, false);
}
//Parse data outside of Vega.
data_format = graph_spec.data[0].format;
graph_spec.data[0].format = null;
if(has_data)
data_values = vg.util.read(graph_spec.data[0].values, data_format);
//Update graph when rendering mode or axis scale is changed.
document.getElementById("renderCanvas").addEventListener('change', function(){
setRenderer(this.value);
......
......@@ -507,6 +507,7 @@
"name": "balance",
"range": "height",
"nice": false,
"clamp": true,
"domain": [0, 100]
}
],
......
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="UTF-8">
<title>Stutter Model Visualisation - FDSTools</title>
<!-- BEGIN_LIBRARIES -->
......@@ -145,7 +146,7 @@
function parse(){
var this_stamp = ++stamp;
vg.parse.spec(graph_spec, function(chart){
// Cancel rendering if a new parse() call was made.
//Cancel rendering if a new parse() call was made.
if(this_stamp != stamp)
return;
......@@ -192,9 +193,11 @@
}
//Load the data (input is a fileList object; only the first file is loaded).
var currentlyLoadedFile = "no/file/loaded";
function loadDataset(fileList){
if(!graph_spec || !fileList || !fileList.length)
if(!graph_spec || !fileList || !fileList.length || fileList[0].name == currentlyLoadedFile)
return;
currentlyLoadedFile = fileList[0].name;
var reader = new FileReader();
reader.onload = function(e){
if(fileList && fileList.length && fileList[0].name)
......@@ -212,11 +215,24 @@
function saveImage(){
var link = document.getElementById("saveLink");
var imageType = document.getElementById("renderSVG").checked? "svg": "png";
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
if(window.navigator.msSaveOrOpenBlob){
//Internet Explorer has its own ways of doing things.
var b;
if(imageType == "svg")
b = new Blob([graph._el.innerHTML], {type: "image/svg+xml;charset=utf-8"});
else
b = graph._el.firstChild.msToBlob();
window.navigator.msSaveOrOpenBlob(b, fileName + "." + imageType);
if(b.msClose)
b.msClose();
}
else{
link.setAttribute("href", graph.toImageURL(imageType));
link.setAttribute("download", fileName + "." + imageType);
link.click();
link.setAttribute("href", "javascript:void(saveImage())");
link.removeAttribute("download");
}
return false;
}
......@@ -236,7 +252,13 @@
document.addEventListener('drop', function(evt){
evt.stopPropagation();
evt.preventDefault();
//Try to clear the currently displayed file, then try to set
//the drag-'n-dropped file as the selected one on the file input.
//Both actions are not supported in all major browsers.
document.getElementById("fileselect").value = "";
document.getElementById("fileselect").files = evt.dataTransfer.files;
loadDataset(evt.dataTransfer.files);
}, false);
}
......
This diff is collapsed.
To-do:
* [!] Additions needed for publication:
* [If not too difficult to implement] BGEstimate should start with homozygous
samples and add heterozygous samples later to optimise correction.
* Summary statistics for BGEstimate based on top N genotypes per marker,
which is the highest percentage remaining background in reference database
(maybe also additional value for confidence interval).
* Visualisation to display highest remaining background (positive and
negative) in known samples after BGCorrect analysis.
* Update bundled D3 to v3.5.12 (now at v3.5.10)
* Libconvert should check whether the reference sequence is the same as one of
the aliases when converting to TSSV format, because it includes the Amel-X
allele twice now.
* Add options to Libconvert to generate a template for STR or non-STR markers.
* BGPredict's get_relative_frequencies may produce a "RuntimeWarning: invalid
value encountered in double_scalars" when computing (with n=0):
A[C[i]] *= stutters[i]["amount"] / A[C[i]].sum()
* BGCorrect might perform better if it 'is given a way out' for those cases
where a sample contains a genuine allele that does not occur in the noise
profiles. I could try to add an all-zero (except for the 100) noise profile
for those missing sequences.
* Warn user in BGEstimate if no samples are given.
* Known bug: regex_longest_match does not give the longest match if a shorter
match is possible and found earlier at the same position.
* Allow loading multiple files into HTML visualisations and provide prev/next
buttons to browse them.
* Add sequence format options back to Samplestats (it turns out to be the only
tool without them).
* In HTML visualisations, before changing a signal value, check whether the
page is scrolled all the way to the right and if so, make sure to scroll to
the right after the change as well.
* Add tool that takes a configuration file and runs a pipeline of other tools.
* Add tool to summarise various statistics about the entire analysis pipeline:
(TODO: Write this list)
* Add tool that takes a configuration file and runs a pipeline of other tools.
* Add "allow_N" flag to [no_repeat] markers. If the flag is specified, the
reference sequence may contain Ns. People might need this for the rCRS mtDNA
reference sequence.