diff --git a/AUTHORS b/AUTHORS
deleted file mode 100644
index 81d02020d8d83361bb7a7c64569504440d643466..0000000000000000000000000000000000000000
--- a/AUTHORS
+++ /dev/null
@@ -1,8 +0,0 @@
-Leiden University Medical Center department of Human Genetics <humgen@lumc.nl>
-Jeroen Laros <j.f.j.laros@lumc.nl>
-Martijn Vermaat <martijn@vermaat.name>
-Jonathan Vis <jvis@liacs.nl>
-Gerben Stouten <gstouten@gmail.com>
-Gerard Schaafsma <g.c.p.schaafsma@lumc.nl>
-Daria Dvorkina <dasha@dvorkina.ru>
-Alisa Muraveva <alisa391213725@yandex.ru>
diff --git a/AUTHORS.rst b/AUTHORS.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0e6450193a5100ae553519953e235af234b2aff6
--- /dev/null
+++ b/AUTHORS.rst
@@ -0,0 +1,23 @@
+Mutalyzer 2 is designed and developed by Jeroen F.J. Laros at the department
+of Human Genetics at Leiden University Medical Center where it is currently
+maintained by Martijn Vermaat.
+
+The following people have worked on developing Mutalyzer:
+
+- Jeroen Laros
+- Martijn Vermaat
+- Jonathan Vis
+- Gerben Stouten
+- Gerard Schaafsma
+- Daria Dvorkina
+- Alisa Muraveva
+
+Specifications are given by Peter E.M. Taschner and Johan T. den Dunnen.
+
+Furthermore we would like to thank the following people for their valuable
+work on previous versions that acted as a guideline for the development of the
+current version:
+
+- Ernest van Ophuizen
+- Martin Wildeman
+- Corinne Bareil
diff --git a/CHANGES.rst b/CHANGES.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d4867e5055c438e4b12a6e72521fc262af3335ef
--- /dev/null
+++ b/CHANGES.rst
@@ -0,0 +1,290 @@
+Changelog
+=========
+
+This is a record of changes made between each Mutalyzer release.
+
+
+Version 2.0.beta-31
+-------------------
+
+Release date to be decided.
+
+
+Version 2.0.beta-30
+-------------------
+
+Released on February 18th 2014.
+
+- Handle NCBI Entrez response validation errors (fixes, among other things,
+ `LOVD#29 <https://humgenprojects.lumc.nl/trac/LOVD3/ticket/29>`_).
+- Loosen error severity when CDS cannot be translated.
+- Mutalyzer development migrated from Subversion to Git for version control.
+
+
+Version 2.0.beta-29
+-------------------
+
+Released on October 11th 2013.
+
+- Add Jonathan Vis attribution and COMMIT logo to about page.
+
+
+Version 2.0.beta-28
+-------------------
+
+Released on September 18th 2013.
+
+- Enable the HTTP/RPC+JSON web service to be used with POST requests.
+
+
+Version 2.0.beta-27
+-------------------
+
+Released on June 18th 2013.
+
+- Fix caching transcript-protein links from NCBI, reducing impact of NCBI
+ communication problems.
+
+
+Version 2.0.beta-26
+-------------------
+
+Released on April 9th 2013.
+
+- Added mm10 (Mouse) transcript mappings to position converter.
+- LRG parser updated to LRG 1.7 schema (`#127
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/127>`_).
+
+
+Version 2.0.beta-25
+-------------------
+
+Released on March 25th 2013.
+
+- Detect incorrect exon annotation in transcript references.
+- Move documentation to Trac.
+- Exon table is included in `runMutalyzer` webservice results.
+- Temporarily disable frameshift detection in experimental description
+ extractor (`#124
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/124>`_).
+- Allow selectors on transcript references in position converter.
+- Syntax checker now supports protein level variant descriptions.
+
+
+Version 2.0.beta-24
+-------------------
+
+Released on December 10th 2012.
+
+- Rename some warning codes (webservice API) (`#98
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/98>`_).
+- Variants on mtDNA in position converter.
+
+
+Version 2.0.beta-23
+-------------------
+
+Released on November 8th 2012.
+
+No user-visible changes.
+
+
+Version 2.0.beta-22
+-------------------
+
+Released on November 2nd 2012.
+
+- Submitting batch jobs via the web services (`#115
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/115>`_).
+- Allow for leading whitespace in batch job input (`#107
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/107>`_).
+- New `descriptionExtract` webservice function.
+- Name checker now includes description extractor output as an experimental
+ service.
+- Slice chromosome by gene name in reference file loader is now case
+ insensitive (`#118
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/118>`_).
+- Warn on missing positioning scheme (`#114
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/114>`_).
+
+
+Version 2.0.beta-21
+-------------------
+
+Released on July 23rd 2012.
+
+- Support compound variants in position converter.
+- Support non-coding transcripts in position converter (`#102
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/102>`_).
+- Move to new RPC library version, causing slight change in HTTP/RPC+JSON
+ webservice output (more wrappers around output), but fixes #104.
+- Fix position converter for delins with explicit deleted sequence.
+- Fix description update from Version 2.0.beta-20 to use- notation instead of
+ counting.
+
+
+Version 2.0.beta-20
+-------------------
+
+Released on July 21st 2012.
+
+- Disabled the ``-u`` and ``+d`` convention in favour of the official HGVS
+ recommendations.
+
+
+Version 2.0.beta-19
+-------------------
+
+Released on June 21st 2012.
+
+- Fix crash on inversions (`#99
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/99>`_).
+
+
+Version 2.0.beta-18
+-------------------
+
+Released on June 7th 2012.
+
+- Moved from soaplib to rpclib for webservices (`#66
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/66>`_).
+- Added HTTP/RPC+JSON webservice (`#18
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/18>`_).
+- Fixed name checker errors in some adjacent variants (`#83
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/83>`_).
+- Name checker form now uses GET requests to support easier linking to result
+ pages.
+- You can now specify chromosomes by name in the reference file loader (`#92
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/92>`_).
+- Made batch daemon not crash on MySQL restarts (`#91
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/91>`_).
+- Position converter now detects incorrect order in position ranges (`#95
+ <https://humgenprojects.lumc.nl/trac/mutalyzer/ticket/95>`_).
+- Added NBIC logo to 'about' page.
+
+
+Version 2.0.beta-17
+-------------------
+
+Released on April 2nd 2012.
+
+- Fixed crossmapping bug for some transcripts.
+- Fixes for NCBI Entrez EFetch Version 2.0 release.
+- Better chromosomal variant descriptions.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-16
+-------------------
+
+Released on March 1st 2012.
+
+- Fixed position converter mapping info for some transcripts.
+- Fixed deletion with deleted sequence length as argument.
+
+
+Version 2.0.beta-15
+-------------------
+
+Released on February 20th 2012.
+
+- Added 'Description Extractor' (see the main menu).
+- Fixes for NCBI Entrez EFetch Version 2.0 release.
+- Added chromosomal positions to `getTranscriptsAndInfo` webservice.
+- Fixed chromosome slicing on reverse complement
+- Fixed describing NOP variants with ``=``.
+- Added Reference sequence info in `runMutalyzer` SOAP function response.
+- Fixed mapping info for genes mapped to more than one chromosome.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-14
+-------------------
+
+Released on January 26th 2012.
+
+- Added a SOAP service `getTranscriptsMapping`.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-13
+-------------------
+
+Released on January 25th 2012.
+
+- Accept EX positioning scheme.
+- Fix handling of LRG reference sequences.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-12
+-------------------
+
+Released on November 25th 2011.
+
+- Accept plasmid reference sequences.
+- View variant position in UCSC Genome Browser (only for transcript
+ references).
+- Retry querying dbSNP if it does not respond the first time.
+- Support reference GenBank files built from contigs.
+- Add optional argument to SOAP service `numberConversion` to map chromosomal
+ locations to any gene.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-11
+-------------------
+
+Released on September 30st 2011.
+
+- Major code refactoring:
+
+ - Mutalyzer is now structured as a proper Python package.
+ - Reworked installation and upgrade procedure.
+ - Remote installation using Fabric.
+ - Batch scheduler is now a proper system daemon.
+ - Use mod_wsgi (with web.py) instead of the deprecated mod_python.
+ - Added a lot of internal documentation.
+ - Introduce unit tests.
+ - Handle deletions of entire exons.
+ - Added a SOAP service `info`.
+ - Handle unknown (fuzzy) intronic positions.
+ - Automatic synchronization of database and cache between Mutalyzer
+ installations.
+ - Use NCBI instead of UCSC for transcript mapping info.
+ - Added a SOAP service `getdbSNPDescriptions`.
+ - Moved Trac and Subversion repository to new server.
+ - Implement HTTP HEAD method for ``/Reference/*`` locations.
+
+- Added a SOAP service `ping`.
+- Added an optional versions parameter to the SOAP service `getTranscripts`.
+- Various smaller features and bugfixes.
+
+
+Version 2.0.beta-10
+-------------------
+
+Released on July 21st 2011.
+
+- Greatly reduce runtime for large batch jobs.
+
+
+Version 2.0.beta-9
+------------------
+
+Released on June 27th 2011.
+
+- Reworked the calculation of new splice site positions.
+- Optionally restrict SOAP service `getTranscriptsAndInfo` transcripts to a
+ gene.
+- Add raw variants to SOAP service `runMutalyzer` results.
+- Provide webservice client examples.
+- Various smaller features and bugfixes.
+
+
+Older versions
+--------------
+
+The first lines of code for Mutalyzer 2.0 where written July 28th 2009, and
+version 2.0.beta-8 was released on January 31st 2011. As far as Mutalyzer 1 is
+concerned, archaeology is not really our field of research.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
deleted file mode 100644
index 09c43ca83f87780f75fcbfc4874de742dcaab79e..0000000000000000000000000000000000000000
--- a/DEVELOPMENT.md
+++ /dev/null
@@ -1,232 +0,0 @@
-Mutalyzer development
-=====================
-
-Development of Mutalyzer happens on the GitLab server:
-https://git.lumc.nl/mutalyzer/mutalyzer
-
-
-Coding style
-------------
-
-In general, try to follow the [PEP 8](http://www.python.org/dev/peps/pep-0008)
-guidelines for Python code and
-[PEP 257](http://www.python.org/dev/peps/pep-0257/) for docstrings.
-
-
-Unit tests
-----------
-
-The unit tests depend on a running batch daemon, webserver, and SOAP
-web service:
-
- sudo /etc/init.d/mutalyzer-batchd start
- sudo /etc/init.d/apache2 start
-
-Now run the tests with:
-
- nosetests -v
-
-Or, if you are in a hurry, skip the long-running tests with:
-
- MUTALYZER_QUICK_TEST=1 nosetests -v
-
-
-Working with feature branches
------------------------------
-
-New features are best implemented in their own branches, isolating the work
-from unrelated developments. In fact, it's good practice to **never work
-directly on the master branch** but always in a separate branch. For this
-reason, the master branch on the GitLab server is locked. Feature branches can
-be merged back into master via a **merge request** in GitLab.
-
-Before starting work on your feature, create a branch for it:
-
- git checkout -b your-feature
-
-Commit changes on this branch. If you're happy with it, push to GitLab:
-
- git push origin your-feature -u
-
-Now create a merge request to discuss the implementation with your
-colleagues. This might involve adding additional commits which are included in
-the merge request by pushing your branch again:
-
- git commit
- git push
-
-You may also be asked to rebase your branch on the master branch if it has
-changed since you started your work. This will require a forced push:
-
- git fetch
- git rebase origin/master
- git push -f
-
-If the work is done, a developer can merge your branch and close the merge
-request. After the branch was merged you can safely delete it:
-
- git branch -d your-feature
-
-
-Release management
-------------------
-
-The current Mutalyzer version is recorded in `mutalyzer/__init__.py`. See the
-comments in that file for more info of the versioning scheme.
-
-On the event of a new release, the following is done:
-
- emacs mutalyzer/__init__.py
-
-Update `__date__`, remove `dev` from `__version_info__` and set `RELEASE` to
-`True`.
-
- git commit -am 'Bump version to 2.0.beta-XX'
- git tag -a 'mutalyzer-2.0.beta-XX'
- git push --tags
-
- emacs mutalyzer/__init__.py
-
-Set `__version_info__` to a new version ending with `'dev'` and set `RELEASE`
-to `FALSE`.
-
- git commit -am 'Open development for 2.0.beta-YY'
-
-Be sure to upgrade your installations to the new version as described in the
-INSTALL file (e.g. `sudo python setup.py develop` for development checkouts).
-
-
-Development notes
------------------
-
-Todo list:
-
-- Improve the web interface design :)
-- Test all uses of mkstemp().
-- Use naming conventions for modules Crossmap, Db, File, GenRecord, Retriever
- and Scheduler.
-- Use standard logging module, with rotating functionality. Race conditions
- on the log file are probably a problem in the current setup.
- Instead of that rotating, we could also use logrotate:
- http://serverfault.com/questions/55610/logrotate-and-open-files
-- Setup continuous integration. Currently, I'm most impressed with Hudson.
- http://hudson-ci.org/
- http://www.rhonabwy.com/wp/2009/11/04/setting-up-a-python-ci-server-with-hudson/
- Or perhaps Jenkins.
- http://jenkins-ci.org/
-- Migrate Javascript to JQuery.
-- I think in the long run, the Output object is not really the way to go. It
- obscures the control flow. The logging part should use the standard logging
- module. The data gathering by the Output object is probably better handled
- by explicitely returning data objects from functions.
-- Migrate from TAL to a more mondern and maintained Python template library,
- for example jinja.
-- Develop a large test suite.
-- Create a web interface url to watch the progress of a batch job.
-- Create web services for the batch jobs (steal ideas from Jeroen's DVD
- web service).
-- Use virtualenv?
-- Use SQLAlchemy?
-- Password for MySQL user.
-- In deployment, remove old versions of Mutalyzer package?
-- Check for os.path.join vulnerabilities.
-- Use a standard solution for the database migrations in extras/migrations.
-- Use something like Sphinx to generate development documentation from code.
-- There are some problems with the batch architecture, especially that there
- cannot be multiple workers without synchronisation problems.
- Good read: http://news.ycombinator.com/item?id=3002861
- Suggestion: http://celeryproject.org/
-- Have a normal 404 page.
-- Maintenance (and/or read-only) mode.
-- Cleanup this document.
-- Be more explicit in all the type of descriptions we don't currently support.
-
-Code style guide:
-
-- Follow PEP 8 (code) and PEP 257 (docstrings).
- http://www.python.org/dev/peps/pep-0008/
- http://www.python.org/dev/peps/pep-0257/
- Read the Google Python Style guide:
- http://google-styleguide.googlecode.com/svn/trunk/pyguide.html
-- Use Epydoc style documentation in docstrings.
-- End class and method definitions with their name as comment.
-- Executables are in the bin/ directory.
-- For examples, check established Python projects:
- http://code.djangoproject.com/browser/django/trunk
- http://twistedmatrix.com/trac/browser/trunk
- https://github.com/webpy/webpy
- https://github.com/mitsuhiko/jinja2
- https://bitbucket.org/mramm/tg-21/src
- http://bazaar.launchpad.net/~bzr-pqm/bzr/bzr.dev/files
- https://github.com/ask/celery
-- A lot of code does not yet adhere to these points, this is an ongoing
- effort.
-
-Obsoleted features:
-
-- On eu.liacs.nl:
- /etc/apache2/mods-enabled/rewrite.load contains a rewrite rule that converts
- "Variant_info.php" to "Variant_info".
- When all LOVD versions are above 2.0-23, this rule can be deleted and the
- rewrite module can be disabled.
-- In the Variant_info() function a substitution on error messages is
- performed.
- When all LOVD versions are above 2.0-23, this check can be deleted.
-
-
-Dependencies
-------------
-
-Mutalyzer depends on the following (Debian/Ubuntu) packages:
-
-- mysql-server >= 5.1
-- python >= 2.6
-- python-mysqldb >= 1.2.2
-- python-biopython >= 1.54
-- python-pyparsing >= 1.5.0
-- python-configobj >= 4.4.0
-- python-magic >= 5.04-2
-- python-psutil >= 0.1.3-1
-- python-xlrd >= 0.6.1-2
-- python-daemon >= 1.5.5
-- python-soappy >= 0.12.0-2
-- python-suds >= 0.3.9-1
-
-The web and SOAP interfaces depend on the following packages:
-
-- apache2 >= 2.2.11
-- libapache2-mod-wsgi >= 2.8
-- python-webpy >= 0.33
-- python-rpclib >= 2.8.0-beta
-- python-simpletal >= 4.1-6
-
-Automatic remote deployment depends on Fabric:
-
-- fabric >= 0.9.0-2
-
-The unit tests depend on the following packages:
-
-- python-nose >= 0.11
-- python-webtest >= 1.2.3
-
-As of 2011-08-23, snakefood reports the following imports from the Mutalyzer
-source code (excluding the standard library imports):
-
- Bio
- MySQLdb
- SOAPpy
- configobj
- daemon
- fabric
- lockfile
- lxml
- magic
- nose
- pyparsing
- setuptools
- simpletal
- rpclib
- suds
- web
- webtest
- xlrd
diff --git a/INSTALL.md b/INSTALL.md
deleted file mode 100644
index a4f7c2ce10940b26506b735f30b1fd6a4eb8f2ea..0000000000000000000000000000000000000000
--- a/INSTALL.md
+++ /dev/null
@@ -1,156 +0,0 @@
-Mutalyzer installation instructions
-===================================
-
-
-Default configuration notes
----------------------------
-
-The instructions in this file are quite specific to the standard Mutalyzer
-environment. This consists of a Debian stable (Squeeze) system with Apache
-and Mutalyzer using its mod_wsgi module. Debian conventions are used
-throughout.
-
-The following is an overview of default locations used by Mutalyzer:
-
- Package files /usr/local/lib/python2.6/dist-packages/...
- Configuration /etc/mutalyzer/config
- Log file /var/log/mutalyzer.log
- Cache directory /var/cache/mutalyzer
- Batchd init script /etc/init.d/mutalyzer-batchd
- Mapping update crontab /etc/cron.d/mutalyzer-mapping-update
- Apache configuration /etc/apache2/conf.d/mutalyzer.conf
- Static website files /var/www/mutalyzer/base
-
-The default database user is 'mutalyzer' with no password and the database
-names are 'mutalyzer', 'hg18', and 'hg19'.
-
-By default, Mutalyzer is exposed under the '/mutalyzer' url by Apache.
-
-All Mutalyzer processes run under the www-data user and files created and/or
-modified by Mutalyzer are owned by this user.
-
-If you have a different environment, or want to customize the default
-locations, you can read through these instructions and modify them to your
-needs.
-
-
-Short version
--------------
-
-Run the following commands:
-
- git clone https://git.lumc.nl/mutalyzer/mutalyzer
- cd mutalyzer
- sudo bash extras/pre-install.sh
- sudo python setup.py install
- sudo bash extras/post-install.sh
- sensible-browser http://localhost/mutalyzer
-
-Or follow the more detailed instructions below.
-
-
-Automated deployment on a remote host
--------------------------------------
-
-For deploying Mutalyzer on a remote (production or testing) host, we recommend
-to automate the steps described below by using Fabric and the included
-fabfile. You need Fabric installed on your local machine:
-
- easy_install fabric
-
-To do a deployment on a server with an existing configured Mutalyzer
-installation:
-
- fab deploy -H server1.mutalyzer.nl
-
-To do a fresh deployment on a new server:
-
- fab deploy:boostrap=yes -H server1.mutalyzer.nl
-
-
-Get Mutalyzer
--------------
-
-Since you are reading this, you can probably skip this step. Otherwise, get
-your hands on a tarball and:
-
- tar -zxvf mutalyzer-XXX.tar.gz
- cd mutalyzer-XXX
-
-Or get the source from GitLab directly:
-
- git clone https://git.lumc.nl/mutalyzer/mutalyzer
- cd mutalyzer
-
-
-Install dependencies
---------------------
-
-If you are on Debian or Ubuntu, you can use the following command to install
-all dependencies:
-
- sudo bash extras/pre-install.sh
-
-Otherwise, install them manually (perhaps have a look in the above script for
-a useful dependency list).
-
-
-Install Mutalyzer
------------------
-
-Mutalyzer can be installed using Python setuptools. For a production
-environment:
-
- sudo python setup.py install
-
-Alternatively, if you want to have a development environment, use:
-
- sudo python setup.py develop
-
-The development environment uses symlinks to this source directory, so you can
-develop directly from here. This command should be re-issued whenever the
-version number of Mutalyzer is updated.
-
-
-Setup Mutalyzer
----------------
-
-This step creates configuration files and populates the database:
-
- sudo bash extras/post-install.sh
-
-You can now edit /etc/mutalyzer/config and /etc/apache2/conf.d/mutalyzer.conf
-to your likings.
-
-
-Test the installation
----------------------
-
-You should always test the installation. The tests (for now at least) need
-the batch daemon and the webserver (the SOAP part) running.
-
-Now run the tests:
-
- nosetests
-
-
-Upgrade Mutalyzer
------------------
-
-Unless you installed Mutalyzer in a development environment as described
-above, you can upgrade Mutalyzer to a new version by running from the source
-directory:
-
- sudo python setup.py install
- sudo bash extras/post-upgrade.sh
-
-If you installed Mutalyzer in a development environment, you don't have to
-do anything usually, except for the following situations.
-
-* If the database has changed, run:
-
- for M in extras/migrations/*.migration; do sudo $M migrate; done
-
-* If the Mutalyzer version has changed, run:
-
- sudo python setup.py develop
\ No newline at end of file
diff --git a/LICENSE.rst b/LICENSE.rst
new file mode 100644
index 0000000000000000000000000000000000000000..21e70bce230dc618160fa3bd4cb4386fa095b337
--- /dev/null
+++ b/LICENSE.rst
@@ -0,0 +1,5 @@
+Copyright (c) 2009-2014 by Leiden University Medical Center and contributors
+(see AUTHORS for details).
+
+Mutalyzer does currently not come with a license. Please contact the authors
+if you need one.
diff --git a/MANIFEST.in b/MANIFEST.in
index 8d1f660b8bb0bf8304ec5b2df7eee623d5b8ee91..8c3bb5ace0fb38fa615e34748b00f80344228714 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,2 @@
-include AUTHORS README.md
+include AUTHORS.rst CHANGES.rst LICENSE.rst README.rst
recursive-include mutalyzer/website/templates *
diff --git a/README.md b/README.md
deleted file mode 100644
index 334fd4a5a08584bda0e05086b3accdaac73962d3..0000000000000000000000000000000000000000
--- a/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-Mutalyzer, an HGVS variant nomenclature checker
-===============================================
-
-The canonical Mutalyzer installation can be found at https://mutalyzer.nl
-
-
-Documentation and issue tracker
--------------------------------
-
-See the doc directory for (possibly outdated) developer documentation and
-presentation slides related to Mutalyzer. See the
-[Mutalyzer Trac](https://humgenprojects.lumc.nl/trac/mutalyzer) for user
-documentation and issue tracker.
-
-
-Installation
-------------
-
-See the INSTALL.md file for installation instructions.
-
-
-Development
------------
-
-The DEVELOPMENT.md file has more information for developers working on
-Mutalyzer.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a93efbe3e934145dc8afda968b1d238fbdb4646b
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,27 @@
+Mutalyzer
+=========
+
+Mutalyzer is an HGVS variant nomenclature checker. The canonical Mutalyzer
+installation can be found at [mutalyzer.nl](https://mutalyzer.nl).
+
+
+Documentation
+-------------
+
+User documentation can be found on the `wiki
+<https://humgenprojects.lumc.nl/trac/mutalyzer>`_. From there, you can also
+submit bug reports and feature requests.
+
+Developer documentation can be found in the ``doc/`` subdirectory and can be
+compiled to HTML by running ``make html`` from there. This requires `Sphinx`_
+to be installed.
+
+
+Copyright
+---------
+
+Mutalyzer does currently not come with a license. Please contact the authors
+if you need one. See the AUTHORS.rst file for a list of authors.
+
+
+.. _Sphinx: http://sphinx-doc.org/
diff --git a/doc/.gitignore b/doc/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..6ec76edf7e4a7cbd64187edf5676da7adaac6c48
--- /dev/null
+++ b/doc/.gitignore
@@ -0,0 +1 @@
+/.build
diff --git a/doc/.static/.gitignore b/doc/.static/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/doc/API/api.conf b/doc/API/api.conf
deleted file mode 100644
index 48ff9466067ca572b591bde1662e462e7780e8f0..0000000000000000000000000000000000000000
--- a/doc/API/api.conf
+++ /dev/null
@@ -1,107 +0,0 @@
-[epydoc] # Epydoc section marker (required by ConfigParser)
-
-# modules
-# The list of objects to document. Objects can be named using
-# dotted names, module filenames, or package directory names.
-# Aliases for this option include "objects" and "values".
-#modules: ../../src/Modules, ../../src/*.py
-modules: ../../src/Modules, ../../src/*.py
-#modules: src/Modules/Db.py, src/Modules/File.py, src/Modules/Config.py
-
-# output
-# The type of output that should be generated. Should be one
-# of: html, text, latex, dvi, ps, pdf.
-output: pdf
-
-# target
-# The path to the output directory. May be relative or absolute.
-target: doc/API/api
-
-# docformat
-# The default markup language for docstrings, for modules that do
-# not define __docformat__. Defaults to epytext.
-docformat: epytext
-
-# css
-# The CSS stylesheet for HTML output. Can be the name of a builtin
-# stylesheet, or the name of a file.
-#css: white
-
-# name
-# The documented project's name.
-name: "Mutalyzer 2.0"
-
-# url
-# The documented project's URL.
-url: http://www.mutalyzer.nl/2.0/
-
-# link
-# HTML code for the project link in the navigation bar. If left
-# unspecified, the project link will be generated based on the
-# project's name and URL.
-#link: <a href="somewhere">My Cool Project</a>
-
-# top
-# The "top" page for the documentation. Can be a URL, the name
-# of a module or class, or one of the special names "trees.html",
-# "indices.html", or "help.html"
-top: ../../src
-
-# help
-# An alternative help file. The named file should contain the
-# body of an HTML file; navigation bars will be added to it.
-#help: my_helpfile.html
-
-# frames
-# Whether or not to include a frames-based table of contents.
-#frames: yes
-
-# private
-# Whether or not to inclue private variables. (Even if included,
-# private variables will be hidden by default.)
-private: yes
-
-# imports
-# Whether or not to list each module's imports.
-imports: no
-
-# verbosity
-# An integer indicating how verbose epydoc should be. The default
-# value is 0; negative values will supress warnings and errors;
-# positive values will give more verbose output.
-verbosity: 1
-
-# parse
-# Whether or not parsing should be used to examine objects.
-parse: yes
-
-# introspect
-# Whether or not introspection should be used to examine objects.
-introspect: yes
-
-# graph
-# The list of graph types that should be automatically included
-# in the output. Graphs are generated using the Graphviz "dot"
-# executable. Graph types include: "classtree", "callgraph",
-# "umlclass". Use "all" to include all graph types
-graph: all
-
-# dotpath
-# The path to the Graphviz "dot" executable, used to generate
-# graphs.
-dotpath: /usr/bin/dot
-
-# sourcecode
-# Whether or not to include syntax highlighted source code in
-# the output (HTML only).
-#sourcecode: yes
-
-# pstat
-# The name of one or more pstat files (generated by the profile
-# or hotshot module). These are used to generate call graphs.
-#pstat: profile.out
-
-# separate-classes
-# Whether each class should be listed in its own section when
-# generating LaTeX or PDF output.
-separate-classes: yes
diff --git a/doc/API/mkapidoc.sh b/doc/API/mkapidoc.sh
deleted file mode 100644
index 9ae6c00f84bf549efc6dc842d54f5d56f5a1c867..0000000000000000000000000000000000000000
--- a/doc/API/mkapidoc.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-epydoc --config api.conf
-mv api/api.pdf .
-rm -rf api/
diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..26e0e92a9893030f50b178a22c9780e3998dd316
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,177 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = .build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " texinfo to make Texinfo files"
+ @echo " info to make Texinfo files and run them through makeinfo"
+ @echo " gettext to make PO message catalogs"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " xml to make Docutils-native XML files"
+ @echo " pseudoxml to make pseudoxml-XML files for display purposes"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Mutalyzer.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Mutalyzer.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/Mutalyzer"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Mutalyzer"
+ @echo "# devhelp"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through platex and dvipdfmx..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo
+ @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+ @echo "Run \`make' in that directory to run these through makeinfo" \
+ "(use \`make info' here to do that automatically)."
+
+info:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo "Running Texinfo files through makeinfo..."
+ make -C $(BUILDDIR)/texinfo info
+ @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+ $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+ @echo
+ @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+ $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+ @echo
+ @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+ $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+ @echo
+ @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/doc/TechnicalReference/TechnicalReference.tex b/doc/TechnicalReference/TechnicalReference.tex
index ec426f28179ecdd2c2131354937b27c5428ae511..0ebc38f26479e208edc6e2bca59e4aa708c0d0ff 100644
--- a/doc/TechnicalReference/TechnicalReference.tex
+++ b/doc/TechnicalReference/TechnicalReference.tex
@@ -26,6 +26,14 @@
\pagenumbering{arabic}
+\section{A word of warning} \label{sec:warning}
+Please note that this document is heavily out of date. Please refer to the
+Sphinx-based documentation in the parent directory and to the comments in the
+source code. Any remaining relevant sections from this document have to be
+moved to the Sphinx-based documentation at some point.
+
+\newpage
+
\section{Introduction} \label{sec:introduction}
This document is intended for developers. It describes the internal workings
of the Mutalyzer \thisversion\ modules and their interactions.
@@ -36,27 +44,27 @@ This document is for internal use only.
\section{Modules} \label{sec:modules}
Mutalyzer \thisversion\ consists of several modules, which can be used in a
-number of combinations. Several possible combinations are described in
+number of combinations. Several possible combinations are described in
Sections~\ref{sec:programs}~and~\ref{sec:interfaces}.
In this section, we give a global overview of all modules and their function,
-for a detailed description of the module interface, please refer to the
+for a detailed description of the module interface, please refer to the
API~\cite{API} documentation.
\subsection{The Config module} \label{subsec:config}
The \texttt{Config} module loads the configuration file
-(\texttt{mutalyzer.conf}) and processes the variables.
+(\texttt{mutalyzer.conf}) and processes the variables.
\begin{itemize}
\item Numbers will be converted to integers.
-\item If necessary, integers will be multiplied by a constant (to convert
+\item If necessary, integers will be multiplied by a constant (to convert
megabytes to bytes for example).
\item Variables are stored in a specific sub class of the \texttt{Config}
- class.
+ class.
\end{itemize}
So all the variables that are associated with the \texttt{Retriever} module for
-example, are stored in the \texttt{Config.Retriever} class. Upon making an
+example, are stored in the \texttt{Config.Retriever} class. Upon making an
instance of the \texttt{Retriever} class, this object can be passed to it to
initialise the class.
@@ -70,7 +78,7 @@ the constructor to make the module aware of the program that constructed it.
This is a way to make a distinction between different programs and interfaces
in the log file.
-Messages are stored in a list of \emph{message objects}. A message object
+Messages are stored in a list of \emph{message objects}. A message object
consists of the fields described in Table~\ref{tab:message}.
\begin{table}[H]
@@ -80,7 +88,7 @@ Field name & Description \\
\hline
origin & Name of the module creating this object. \\
level & Importance of the message (see Table~\ref{tab:messagelevel}). \\
-code & The error code of the message
+code & The error code of the message
(see Appendix~\ref{subsec:error}). \\
description & A description of the message.
\end{tabular}
@@ -89,7 +97,7 @@ description & A description of the message.
\end{table}
In the message object, the \emph{file name} field refers to the program in
-which the message was generated, the \emph{severity level}, as described in
+which the message was generated, the \emph{severity level}, as described in
Table~\ref{tab:messagelevel} indicates the class of the message, ranging from a
log message to to fatal error, the \emph{code} field refers to the type of
message and the \emph{description} field is used for a long descriptive
@@ -112,19 +120,19 @@ $5$ & Off & Show nothing. \\
\end{table}
The \emph{code} field can be used by a program that uses this module as a means to alter the control flow, instead of parsing the description (that is likely
-to change in later versions). The codes and their meaning can be found in
+to change in later versions). The codes and their meaning can be found in
Appendix~\ref{subsec:error}.
The \emph{severity level} field is used both for logging as well as for
retrieving messages, both ways are controlled by configuration variables that
-are passed to the \texttt{Output} module at the time of construction.
+are passed to the \texttt{Output} module at the time of construction.
The \texttt{outputlevel} variable is used by the \texttt{getMessages()} member
function. All messages of which the level exceeds the value of this variable
will be returned as a list. Logging is controlled by the \texttt{loglevel}
variable. All messages of which the level exceeds the values of this variable
will be logged. Additionally, the level $-1$ is reserved for logging
exclusively. Messages that are generated with this level, will not appear when
-using the \texttt{getMessages()} function.
+using the \texttt{getMessages()} function.
In Table~\ref{tab:outputconfig} all configurable settings of this module are
described.
@@ -139,21 +147,21 @@ datestring & Prefix for each log message. \\
loglevel & Level at which messages are logged. \\
outputlevel & Level at which messages are returned by \texttt{getMessages()}.
\end{tabular}
-\caption{Configuration variables of the \texttt{Output} object}
+\caption{Configuration variables of the \texttt{Output} object}
\label{tab:outputconfig}
\end{center}
\end{table}
-For storage of output, a dictionary is used. The values in this dictionary
+For storage of output, a dictionary is used. The values in this dictionary
are lists. A pair (key, value) can be set by using the \texttt{addOutput()}
function. Subsequent calls to this function with the same key as parameter,
will expand the list that is pointed to by the key.
For retrieval of output, the \texttt{getOutput()} and
-\texttt{getIndexedOutput()} functions can be used. Both functions require a
+\texttt{getIndexedOutput()} functions can be used. Both functions require a
key name as argument. The first one returns the entire list pointed at by the
key, while the second one only returns one element of this list. The index of
-this element is passed as a parameter. If either the key or (in the latter
+this element is passed as a parameter. If either the key or (in the latter
case) the index does not exist, an empty list is returned.
\subsection{The Db module} \label{subsec:db}
@@ -181,9 +189,9 @@ The \texttt{Db} module has the following derived classes:
The \texttt{Mapping} class provides an interface for databases that contain
mapping information. At the time of this writing, mapping information of
-human builds \emph{hg18} and \emph{hg19}~\cite{GN} are available. This class
+human builds \emph{hg18} and \emph{hg19}~\cite{GN} are available. This class
provides functions to convert chromosomal positions into transcript oriented
-positions and vice versa. These conversions are needed for the mapping of
+positions and vice versa. These conversions are needed for the mapping of
data in locus specific databases~\cite{LOVD} (from transcript level to
chromosome level), or for the analysis of variants discovered in Next
Generation Sequencing projects (in which case the conversion is likely to be
@@ -213,16 +221,16 @@ Variable & Description \\
\hline
internalDb & Name of the internal database. \\
dbNames & List of MySQL mapping database names. \\
-LocalMySQLuser & MySQL username for the local databases (internalDb and
+LocalMySQLuser & MySQL username for the local databases (internalDb and
dbNames). \\
LocalMySQLhost & Host name for the local databases. \\
RemoteMySQLuser & MySQL username for the UCSC database. \\
RemoteMySQLhost & Host name for the UCSC database. \\
-UpdateInterval & Retrieve all entries modified within a certain number of
+UpdateInterval & Retrieve all entries modified within a certain number of
days. \\
TempFile & Temporary file for updated UCSC mapping information.
\end{tabular}
-\caption{Configuration variables of the \texttt{Db} object}
+\caption{Configuration variables of the \texttt{Db} object}
\label{tab:dbconfig}
\end{center}
\end{table}
@@ -243,7 +251,7 @@ within a reference sequence file has changed (without changing anything
crucial), so the version number of this retrieved file was not increased. An
other useful function is one that returns a new \texttt{UD} number. \texttt{UD}
numbers are assigned to files that are local to the Mutalyzer installation.
-They will also be discussed in Section~\ref{subsubsec:genbankretriever}.
+They will also be discussed in Section~\ref{subsubsec:genbankretriever}.
The \texttt{Retriever} class is configured with variables shown in
Table~\ref{tab:retrieverconfig}.
@@ -260,7 +268,7 @@ maxDldSize & The maximum size of a downloaded reference sequence file in
megabytes. \\
minDldSize & The minimum size of a downloaded reference sequence file in bytes.
\end{tabular}
-\caption{Configuration variables of the \texttt{Retriever} object}
+\caption{Configuration variables of the \texttt{Retriever} object}
\label{tab:retrieverconfig}
\end{center}
\end{table}
@@ -268,7 +276,7 @@ minDldSize & The minimum size of a downloaded reference sequence file in bytes.
\subsubsection{The GenBankRetriever class} \label{subsubsec:genbankretriever}
The \texttt{GenBankRetriever} class provides functions to retrieve a
\emph{GenBank} file from the NCBI~\cite{NCBI}. There are several interfaces
-that accomplish this task:
+that accomplish this task:
\begin{itemize}
\item Automatic retrieval.
@@ -295,9 +303,9 @@ variable). Then, the \texttt{efetch()} function from the Biopython package can
be used to retrieve the sequence of interest. The retrieved slice is placed in
the cache and is given its unique \texttt{UD} number. After retrieval, the
md5sum is calculated and stored in the internal database together with the
-accession number of the chromosome or contig, the \texttt{UD} number, the
+accession number of the chromosome or contig, the \texttt{UD} number, the
orientation and the positions used to make the slice in the \texttt{GBInfo}
-table.
+table.
Retrieval by gene name can only be done for builds of genomes that are
considered current by the NCBI. At the time of this writing, for example, a
@@ -328,18 +336,18 @@ After successful retrieval, the file is given a \texttt{UD} number and is
placed in the cache directory. The md5sum of the file, as well as its
\texttt{UD} number and the original hyperlink are stored in the database.
-Each interface stores information about the origin and the way of retrieval
+Each interface stores information about the origin and the way of retrieval
in the local database in the \texttt{GBInfo} table. This is done to ease the
task of retrieving a deleted file from the cache. If for example, an
\texttt{UD} number of a deleted file is given, the database can be queried to
see if there is still enough information to re-retrieve the reference sequence
file. If the way of retrieval was by slicing (and thus also by retrieving via
-gene symbol), the coordinates and orientation, together with the accession
+gene symbol), the coordinates and orientation, together with the accession
number of the chromosome or contig can be used to make the \texttt{UD}
available again. If necessary, the md5sum is updated.
When a file is deleted where the method of retrieval was via a hyperlink, the
-file is downloaded again, but is only revived if and only if the md5sum
+file is downloaded again, but is only revived if and only if the md5sum
matches. Manually uploaded files can not be revived.
An other use of the md5sum is preventing multiple uploads. If the md5sum is
@@ -356,11 +364,11 @@ The \texttt{Mutator} module provides a class that has a number of standard
mutation functions like \texttt{delM()}, \texttt{insM()}, etc. which are all
wrappers for the private \texttt{\_\_mutate()} member function. This function
operates under the assumption that every mutation can be written as a
-\emph{deletion-insertion}. The wrapper functions convert positions to
+\emph{deletion-insertion}. The wrapper functions convert positions to
\emph{interbase coordinates} when needed and call the \texttt{\_\_mutate()}
-function. Each wrapper function stores a triple (descriptive message, old
-sequence, new sequence) in the \texttt{Output} object under the name
-\texttt{visualisation}. The old- and new sequence are returned by the
+function. Each wrapper function stores a triple (descriptive message, old
+sequence, new sequence) in the \texttt{Output} object under the name
+\texttt{visualisation}. The old- and new sequence are returned by the
\texttt{\_\_mutate()} function.
The \texttt{\_\_mutate()} function performs the following tasks:
@@ -389,14 +397,14 @@ maxvissize & Maximum length of visualised mutations. \\
flankclipsize & Length of the flanking sequences of the clipped mutations (see
maxvissize).
\end{tabular}
-\caption{Configuration variables of the \texttt{Mutator} object}
+\caption{Configuration variables of the \texttt{Mutator} object}
\label{tab:mutatorconfig}
\end{center}
\end{table}
-If the original or mutated part of a sequence exceeds the \texttt{maxvissize}
+If the original or mutated part of a sequence exceeds the \texttt{maxvissize}
configuration value, this subsequence is broken up in three parts. The outer
-parts consist of the head and tail of the subsequence (of length
+parts consist of the head and tail of the subsequence (of length
\texttt{flankclipsize}) and the center part of the subsequence is replaced by
a number to indicate the length of the subsequence that is not shown. Now the
shortest subsequence is padded to align the flanking sequences vertically.
@@ -418,7 +426,7 @@ special care must be taken when mutating a sequence. A deletion or insertion
will result in a position shift that will alter the coordinates of all
downstream mutations. Therefore, when a mutation is requested (by calling the
\texttt{\_\_mutate()} function), the coordinates are converted to the correct
-ones. This is done by registering each position where a \emph{shift} occurs
+ones. This is done by registering each position where a \emph{shift} occurs
and the length and direction of this shift. By iterating through this list, the
correct coordinate can be calculated (iteration stops when we find a shift
position that is larger than the requested position).
@@ -439,9 +447,9 @@ resultsDir & Location of the results. \\
PIDfile & Location of the PID file. \\
nameCheckOutHeader & The output header for NameChecking \\
syntaxCheckOutHeader & The output header for NameChecking \\
-positionConverterOutHeader & The output header for NameChecking
+positionConverterOutHeader & The output header for NameChecking
\end{tabular}
-\caption{Configuration variables of the \texttt{Scheduler} object}
+\caption{Configuration variables of the \texttt{Scheduler} object}
\label{tab:schedulerconfig}
\end{center}
\end{table}
@@ -464,7 +472,7 @@ bufSize & Amount of bytes to be read for determining the file type. \\
header & The obligatory header in batch request files. \\
tempDir & Directory for temporary files.
\end{tabular}
-\caption{Configuration variables of the \texttt{File} object}
+\caption{Configuration variables of the \texttt{File} object}
\label{tab:fileconfig}
\end{center}
\end{table}
@@ -481,7 +489,7 @@ function is used. This also uses the \texttt{bufSize} configuration variable to
read part of a stream and subsequently guess its file type. This function
returns two values; the \emph{mime type} and a \emph{description}. The latter
is sometimes needed when a container file format is used (as is the case with
-OpenOffice documents). Once the file type is successfully determined, the
+OpenOffice documents). Once the file type is successfully determined, the
appropriate parser is selected.
\subsection{The GenRecord module} \label{subsec:genrecord}
@@ -492,14 +500,14 @@ appropriate parser is selected.
\begin{tabular}{l|p{9cm}}
Variable & Description \\
\hline
-upstream & Number of upstream nucleotides when searching for a
- transcript. \\
-downstream & Number of downstream nucleotides when searching for a
- transcript. \\
-spliceAlarm & \\
+upstream & Number of upstream nucleotides when searching for a
+ transcript. \\
+downstream & Number of downstream nucleotides when searching for a
+ transcript. \\
+spliceAlarm & \\
spliceWarn &
\end{tabular}
-\caption{Configuration variables of the \texttt{GenRecord} object}
+\caption{Configuration variables of the \texttt{GenRecord} object}
\label{tab:genrecordconfig}
\end{center}
\end{table}
@@ -548,25 +556,25 @@ convert from a \emph{genomic coordinate} to a \emph{transcript oriented}
coordinate or a \emph{coding sequence oriented} coordinate. In the remainder
of this section we shall refer to genomic coordinates as \emph{g.}, transcript oriented as \emph{n.} and coding sequence oriented as \emph{c.} coordinates.
Coordinate conversion is done by linking each splice site to either an n. or
-g. coordinate.
+g. coordinate.
The class is initialised by passing an RNA list, a CDS list and an orientation
of the transcript to the constructor. If a CDS is given, the resulting
crossmapper will convert from g. to c. coordinates. If no CDS is given, it will
-convert g. to n. coordinates.
+convert g. to n. coordinates.
Most internal functions are operators that deal with mathematical exceptions
when using the ``biological'' coordinates. The number $0$ does not exist for
example, making $-1$ and $1$ neighbouring integers, which makes calculations
awkward. These internal functions take care of these problems.
-When a crossmapper is made that converts from g. to c. (a CDS was given at
-construction time), the \emph{CDS stop} coordinate in c. notation is stored
+When a crossmapper is made that converts from g. to c. (a CDS was given at
+construction time), the \emph{CDS stop} coordinate in c. notation is stored
separately. Internally the crossmapper does not work with ``star'' (*)
positions for positions between CDS stop and \emph{transcription stop}. The
conversion to ``star''-notation is done at the very end.
-After initialisation, two main functions are available, \texttt{g2x} and
+After initialisation, two main functions are available, \texttt{g2x} and
\texttt{x2g}. Their behaviour depends upon the initialisation of the class and
is described in Table~\ref{tab:behaviour}.
@@ -593,10 +601,10 @@ The second element of the tuple $b$ is the ``offset'' coordinate, this
coordinate refers to the distance to the closest \emph{splice site}. A tuple
$(21, -2)$ for example, means that there is a splice site at position $21$ and
that the coordinate of interest lies $2$ positions before this splice site (in
-an \emph{intron}).
+an \emph{intron}).
-As mentioned before, this function does not take CDS stop into account. It is
-not concerned with positions upstream or downstream of the transcript either.
+As mentioned before, this function does not take CDS stop into account. It is
+not concerned with positions upstream or downstream of the transcript either.
These final conversions are discussed further on in this section.
The function \texttt{x2g()} takes a tuple $(a, b)$ as argument (discussed
@@ -606,7 +614,7 @@ The most important wrapper functions of the \texttt{Crossmap} class are:
\begin{itemize}
\item \texttt{int2main()} converts a $a$ coordinate to a string in
``star''-notation.
-\item \texttt{main2int()} converts a string in ``star''-notation to the $a$
+\item \texttt{main2int()} converts a string in ``star''-notation to the $a$
coordinate.
\item \texttt{int2offset()} converts the $b$ coordinate to an
``offset''-notation, adding an ``u'' for offsets upstream of the
@@ -617,7 +625,7 @@ The most important wrapper functions of the \texttt{Crossmap} class are:
Other wrapper functions in this class only call the wrappers described above.
-Finally, there are a couple of functions that offer more information of a
+Finally, there are a couple of functions that offer more information of a
transcript; an \texttt{info()} function that gives information about
transcription start, transcription stop and CDS stop, a function
\texttt{getSpliceSite()} that returns the genomic coordinate of a splice site
@@ -632,13 +640,13 @@ modules.
\subsection{Mutalyzer} \label{subsec:mutalyzer}
The \texttt{Mutalyzer} program (also known as the \texttt{name checker} perform
several tasks; \emph{semantic checking}, \emph{position conversion},
-\emph{visualisation} and \emph{effect prediction} to some extent.
+\emph{visualisation} and \emph{effect prediction} to some extent.
%TODO more
\subsubsection{Semantic checks}
The main function of \texttt{Mutalyzer} program, is \emph{semantic name
checking}. It uses the \texttt{Parser} module for syntactic checking and the
-generation of a parse tree, and applies the given description to a reference
+generation of a parse tree, and applies the given description to a reference
sequence. This reference sequence is provided by either the \texttt{LRGparser}
of \texttt{GBparser} module. By combining the variant description and the
interpreted reference sequence file, semantic checking can be done.
@@ -664,7 +672,7 @@ to this kind of ambiguity.
Every description can be written as a deletion/insertion (\emph{delins})
operation, but using this representation is not necessarily the shortest one. A
delins of one nucleotide, for example should be written as a substitution.
-Several checks are implemented to reduce one representation to an other. In
+Several checks are implemented to reduce one representation to an other. In
Table~\ref{tab:erosion} a list of possibly ``disguised'' variants is shown.
\begin{table}[H]
@@ -674,7 +682,7 @@ Variant type & Could be \\
\hline
delins & del, ins, dup, subst, inv \\
ins & dup \\
-inv & subst
+inv & subst
\end{tabular}
\end{center}
\caption{Disguised variants.} \label{tab:erosion}
@@ -683,7 +691,7 @@ inv & subst
Apart from this possible disguise, a variant can be described using too much
nucleotides, or a range that is too large. A delins can include part of the
flanking sequence of the real change. To prevent this, the \emph{longest common
-prefix} and \emph{longest common suffix} are pruned. Likewise, an inversion
+prefix} and \emph{longest common suffix} are pruned. Likewise, an inversion
can be too large when the \emph{reverse complement} of a prefix is a suffix.
This is pruned in a similar way. Finally, a delins, subst or inv can have no
effect at all.
@@ -696,7 +704,7 @@ nucleotides is given, this stretch must match the reference sequence.
The effect prediction is very limited. \texttt{Mutalyzer} is able to detect
variants that hit a splice site and a start codon and issue a warning
accordingly. From the generated variant description however, some effects can
-be deduced. A variant that is at position $x+2$ for example, has a high
+be deduced. A variant that is at position $x+2$ for example, has a high
probability of destroying a splice site.
In case of a variant in the start codon, a warning is given, but in case the
@@ -713,14 +721,14 @@ There are two major cases to distinguish:
\end{itemize}
To distinguish between these cases, a combination of the \texttt{CrossMap}
-and \texttt{Mutator} functions is used.
+and \texttt{Mutator} functions is used.
\begin{itemize} % FIXME Is this true? Not just CDS len?
-\item Take the original CDS stop variable, located in the \texttt{CrossMap}
+\item Take the original CDS stop variable, located in the \texttt{CrossMap}
object.
\item Convert this to a genomic position (also using the \texttt{CrossMap}
module.
-\item Use the \texttt{Mutator} module to get the location of the CDS stop
+\item Use the \texttt{Mutator} module to get the location of the CDS stop
variable in the new (mutated) coordinate system.
\item Convert this new position back to a CDS oriented position.
\end{itemize}
@@ -737,10 +745,10 @@ If the reading frame is not disrupted, we first calculate the \emph{longest
common prefix}, remove this from both the original and mutated protein and then
by calculating and removing the \emph{longest common suffix}. The string that
is left, is in general a delins, but can often be simplified by using a number
-of rules. These rules are applied in the \texttt{findInFrameDescription()}
+of rules. These rules are applied in the \texttt{findInFrameDescription()}
function.
-Also, effects on restriction sites are reported, as described in
+Also, effects on restriction sites are reported, as described in
Section~\ref{subsec:mutator}
Both of these functions are wrapped by the \texttt{\_\_toProtDescr()} function
@@ -754,11 +762,11 @@ used later on for visualisation.
\subsubsection{Visualisation} \label{subsubsec:visualisation}
To aid in the analysis of both variants on DNA as well as the derived protein
description, a number of visualisations are made. First of all, all raw
-variants are shown as described in Section~\ref{subsec:mutator}.
+variants are shown as described in Section~\ref{subsec:mutator}.
Apart from this, both the original and the mutated protein are shown and the
differences between them are highlighted. This is done by using the positions
-returned by the \texttt{\_\_toProtDescr()} function as described in
+returned by the \texttt{\_\_toProtDescr()} function as described in
Section~\ref{subsubsec:effect}.
\subsection{VarInfo} \label{subsec:varinfo}
diff --git a/doc/admin.rst b/doc/admin.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e169430b477a21cc692197912663f61fa297fc29
--- /dev/null
+++ b/doc/admin.rst
@@ -0,0 +1,150 @@
+.. highlight:: none
+
+.. _admin:
+
+Administration
+==============
+
+For administrative tasks, Mutalyzer comes with the ``mutalyzer-admin`` command
+line utility. Use its ``-h`` argument in combination with any subcommand for
+detailed usage information, for example::
+
+ $ mutalyzer-admin setup-database -h
+ usage: mutalyzer-admin setup-database [-h] [--destructive] [-c ALEMBIC_CONFIG]
+
+ Setup database tables (if they do not yet exist).
+
+ optional arguments:
+ -h, --help show this help message and exit
+ --destructive delete any existing tables and data
+ -c ALEMBIC_CONFIG, --alembic-config ALEMBIC_CONFIG
+ path to Alembic configuration file
+
+ If Alembic config is given (--alembic-config), this also prepares the database
+ for future migrations with Alembic (recommended).
+
+
+Managing genome assemblies
+--------------------------
+
+Mutalyzer can be loaded with any number of genome assemblies. Each assembly
+includes a list of chromosomes. To list the currently loaded genome
+assemblies::
+
+ $ mutalyzer-admin assemblies list
+ GRCh37 (hg19), Homo sapiens (9606)
+ GRCm38 (mm10), Mus musculus (10090)
+
+Loading a new genome assembly is done with information in a JSON file, of
+which there are some examples in the Mutalyzer source tree under the
+``extras/assemblies`` directory, for example::
+
+ $ mutalyzer-admin assemblies add extras/assemblies/GRCh37.json
+
+For any genome assembly, transcript mappings can be imported. These include
+genomic coordinate mappings of their CDS and exons. Currently, three sources
+of transcript mappings are supported.
+
+.. note:: The following ``mutalyzer-admin assemblies`` subcommands all accept
+ an optional ``--assembly`` argument to specify the genome assembly
+ to import to.
+
+
+Import mappings from an NCBI mapview file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The NCBI provides FTP downloads of transcript mappings for a large number of
+genome assemblies as used by their `Map Viewer
+<http://www.ncbi.nlm.nih.gov/mapview/>`_ service. These can be imported with
+``mutalyzer-admin``, but only after sorting by the *feature_id* and
+*chromosome* columns.
+
+For example, to import transcript mappings for the GRCh37 assembly, run the
+following::
+
+ $ wget ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.37.2/mapview/seq_gene.md.gz
+ $ zcat seq_gene.md.gz | sort -k 11,11 -k 2,2 > seq_gene.sorted.md
+ $ mutalyzer-admin assemblies import-mapview seq_gene.sorted.md 'GRCh37.p2-Primary Assembly'
+
+.. note:: The last argument, ``GRCh37.p2-Primary Assembly``, defines the group
+ label to filter the file on. You would usually want to include it.
+
+
+Import mappings from the UCSC Genome Browser MySQL database
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Transcript mappings from the `UCSC Genome Browser MySQL database
+<https://genome.ucsc.edu/goldenPath/help/mysql.html>`_ can be imported on a
+per-gene basis. This is useful when the NCBI mappings do not (yet) include a
+certain gene or transcript.
+
+For example, to import all TTN transcript mappings::
+
+ $ mutalyzer-admin assemblies import-gene
+
+.. note:: This subcommand chooses the UCSC genome assembly by using the alias
+ of the specified Mutalyzer genome assembly (`hg19` by default).
+
+
+Import mappings from a reference file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For transcript mappings that are not available from our usual sources,
+importing from a genomic reference is supported::
+
+ $ mutalyzer-admin assemblies import-reference NC_012920.1
+
+.. note:: Currently this subcommand is restricted to importing mtDNA
+ transcripts, since it has the chromosome hard coded and only
+ supports one exon per transcript.
+
+
+Showing announcements to users
+------------------------------
+
+It is possible to define an announcement to be shown on the website
+interface. For example, to display *Hello World!* with a link to the `GNU
+Hello World! <http://www.gnu.org/fun/jokes/helloworld.html>`_ page::
+
+ $ mutalyzer-admin announcement set 'Hello World!' \
+ --url http://www.gnu.org/fun/jokes/helloworld.html
+
+To remove the announcement, use ``unset``::
+
+ $ mutalyzer-admin announcement unset
+
+
+Synchronizing the cache with other installations
+------------------------------------------------
+
+Using the ``sync-cache`` subcommand, the reference file cache of a remote
+Mutalyzer installation can be queried for new entries which are then retrieved
+and added to the local cache.
+
+The primary purpose for this is synchronizing reference files loaded by users
+with the reference file loader between different servers. These reference
+files are assigned a unique accession number (starting with ``UD_``) upon
+creation, which is at that point unknown to any other Mutalyzer server.
+
+For example, to synchronize the local reference file cache with the `primary
+Mutalyzer server <https://mutalyzer.nl/>`_::
+
+ $ mutalyzer-admin sync-cache 'https://mutalyzer.nl/services/?wsdl' \
+ 'https://mutalyzer.nl/Reference/{file}'
+
+
+Mutalyzer database setup
+------------------------
+
+After installation, a database needs to be setup for Mutalyzer to run (see
+:ref:`install-setup`)::
+
+ $ mutalyzer-admin setup-database --alembic-config migrations/alembic.ini
+
+The ``--alembic-config`` argument points to the ``alembic.ini`` file in the
+Mutalyzer source tree and it enables initialization of database migration
+management. It is recommended to include it, but you don't need it if you
+don't plan to ever upgrade your Mutalyzer installation.
+
+This subcommand also takes an optional ``--destructive`` argument, which can
+be used to remove any existing database content.
diff --git a/doc/api.rst b/doc/api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6e1524ff00b46000d752950bc17be393ee674528
--- /dev/null
+++ b/doc/api.rst
@@ -0,0 +1,283 @@
+API reference
+=============
+
+.. note:: This reference is incomplete. In particular, modules without
+ reStructuredText formatted docstrings are omitted.
+
+
+.. mutalyzer
+ ---------
+
+ .. automodule:: mutalyzer
+ :members: NOMENCLATURE_VERSION_INFO, NOMENCLATURE_VERSION, COPYRIGHT_YEARS, SOAP_NAMESPACE
+
+
+mutalyzer.announce
+------------------
+
+.. automodule:: mutalyzer.announce
+ :members:
+
+
+mutalyzer.config
+----------------
+
+.. automodule:: mutalyzer.config
+ :members:
+
+
+.. mutalyzer.config.default_settings
+ ---------------------------------
+
+ .. automodule:: mutalyzer.config.default_settings
+ :members:
+
+
+.. mutalyzer.Crossmap
+ ------------------
+
+ .. automodule:: mutalyzer.Crossmap
+ :members:
+
+
+mutalyzer.db
+------------
+
+.. automodule:: mutalyzer.db
+ :members:
+
+
+mutalyzer.db.models
+-------------------
+
+.. I have no idea why, but somehow :members: includes `or_` and `relationship`
+ from SQLAlchemy (which are imported in the module).
+
+.. automodule:: mutalyzer.db.models
+ :members:
+ :exclude-members: or_, relationship
+
+
+mutalyzer.db.queries
+--------------------
+
+.. automodule:: mutalyzer.db.queries
+ :members:
+ :exclude-members: and_, or_
+
+
+.. mutalyzer.describe
+ ------------------
+
+ .. automodule:: mutalyzer.describe
+ :members:
+
+
+.. mutalyzer.entrypoints
+ ---------------------
+
+ .. automodule:: mutalyzer.entrypoints
+ :members:
+
+
+mutalyzer.entrypoints.admin
+---------------------------
+
+.. automodule:: mutalyzer.entrypoints.admin
+ :members:
+
+
+mutalyzer.entrypoints.batch_processor
+-------------------------------------
+
+.. automodule:: mutalyzer.entrypoints.batch_processor
+ :members:
+
+
+mutalyzer.entrypoints.mutalyzer
+-------------------------------
+
+.. automodule:: mutalyzer.entrypoints.mutalyzer
+ :members:
+
+
+mutalyzer.entrypoints.service_json
+----------------------------------
+
+.. automodule:: mutalyzer.entrypoints.service_json
+ :members:
+
+
+mutalyzer.entrypoints.service_soap
+----------------------------------
+
+.. automodule:: mutalyzer.entrypoints.service_soap
+ :members:
+
+
+mutalyzer.entrypoints.website
+-----------------------------
+
+.. automodule:: mutalyzer.entrypoints.website
+ :members:
+
+
+.. mutalyzer.File
+ --------------
+
+ .. automodule:: mutalyzer.File
+ :members:
+
+
+.. mutalyzer.GenRecord
+ -------------------
+
+ .. automodule:: mutalyzer.GenRecord
+ :members:
+
+
+.. mutalyzer.grammar
+ -----------------
+
+ .. automodule:: mutalyzer.grammar
+ :members:
+
+
+.. mutalyzer.mapping
+ -----------------
+
+ .. automodule:: mutalyzer.mapping
+ :members:
+
+
+.. mutalyzer.models
+ ----------------
+
+ .. automodule:: mutalyzer.models
+ :members:
+
+
+.. mutalyzer.mutator
+ -----------------
+
+ .. automodule:: mutalyzer.mutator
+ :members:
+
+
+.. mutalyzer.output
+ ----------------
+
+ .. automodule:: mutalyzer.output
+ :members:
+
+
+.. mutalyzer.parsers
+ -----------------
+
+ .. automodule:: mutalyzer.parsers
+ :members:
+
+
+.. mutalyzer.parsers.genbank
+ -------------------------
+
+ .. automodule:: mutalyzer.parsers.genbank
+ :members:
+
+
+.. mutalyzer.parsers.lrg
+ ---------------------
+
+ .. automodule:: mutalyzer.parsers.lrg
+ :members:
+
+
+mutalyzer.redisclient
+---------------------
+
+.. automodule:: mutalyzer.redisclient
+ :members:
+
+
+.. mutalyzer.Retriever
+ -------------------
+
+ .. automodule:: mutalyzer.Retriever
+ :members:
+
+
+.. mutalyzer.Scheduler
+ -------------------
+
+ .. automodule:: mutalyzer.Scheduler
+ :members:
+
+
+.. mutalyzer.services
+ ------------------
+
+ .. automodule:: mutalyzer.services
+ :members:
+
+
+mutalyzer.services.json
+-----------------------
+
+.. automodule:: mutalyzer.services.json
+ :members:
+
+
+.. mutalyzer.services.rpc
+ ----------------------
+
+ .. automodule:: mutalyzer.services.rpc
+ :members:
+
+
+mutalyzer.services.soap
+-----------------------
+
+.. automodule:: mutalyzer.services.soap
+ :members:
+
+
+mutalyzer.stats
+---------------
+
+.. automodule:: mutalyzer.stats
+ :members:
+
+
+mutalyzer.sync
+--------------
+
+.. automodule:: mutalyzer.sync
+ :members:
+
+
+.. mutalyzer.util
+ --------------
+
+ .. automodule:: mutalyzer.util
+ :members:
+
+
+.. mutalyzer.variantchecker
+ ------------------------
+
+ .. automodule:: mutalyzer.variantchecker
+ :members:
+
+
+mutalyzer.website
+-----------------
+
+.. automodule:: mutalyzer.website
+ :members:
+
+
+mutalyzer.website.views
+-----------------------
+
+.. automodule:: mutalyzer.website.views
+ :members:
diff --git a/doc/changelog.rst b/doc/changelog.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d9e113ec685b92b018297dd2f200341d1fba01f9
--- /dev/null
+++ b/doc/changelog.rst
@@ -0,0 +1 @@
+.. include:: ../CHANGES.rst
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..12b7efa30f666c007258f0a900753bb0c1bfd316
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,275 @@
+# -*- coding: utf-8 -*-
+#
+# Mutalyzer documentation build configuration file, created by
+# sphinx-quickstart on Mon Feb 17 11:01:56 2014.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration ------------------------------------------------
+
+import mutalyzer
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.todo',
+ 'sphinx.ext.viewcode',
+]
+
+# Include todo directives.
+todo_include_todos = False
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['.templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Mutalyzer'
+copyright = u'%s, %s' % ('%d-%d' % mutalyzer.COPYRIGHT_YEARS, mutalyzer.__author__)
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '.'.join(mutalyzer.__version__.split('.')[:2])
+# The full version, including alpha/beta/rc tags.
+release = mutalyzer.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['.build']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# on_rtd is whether we are on readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if not on_rtd: # only import and set the theme if we're building docs locally
+ import sphinx_rtd_theme
+ html_theme = 'sphinx_rtd_theme'
+ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['.static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Mutalyzerdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ ('index', 'Mutalyzer.tex', u'Mutalyzer Documentation',
+ u'Leiden University Medical Center', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'mutalyzer', u'Mutalyzer Documentation',
+ [u'Leiden University Medical Center'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ ('index', 'Mutalyzer', u'Mutalyzer Documentation',
+ u'Leiden University Medical Center', 'Mutalyzer',
+ 'HGVS variant nomenclature checker.', 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
diff --git a/doc/config.rst b/doc/config.rst
new file mode 100644
index 0000000000000000000000000000000000000000..dea8a8b38b85ac72464ca59199ec7f9968d1805f
--- /dev/null
+++ b/doc/config.rst
@@ -0,0 +1,92 @@
+.. highlight:: none
+
+.. _config:
+
+Configuration
+=============
+
+This section describes how to configure Mutalyzer and includes a list of all
+available configuration settings.
+
+Mutalyzer looks for its configuration in the file specified by the
+``MUTALYZER_SETTINGS`` environment variable. Make sure to always have this
+environment variable set when invoking any component of Mutalyzer. One way of
+doing this is by exporting it::
+
+ $ export MUTALYZER_SETTINGS=~/mutalyzer/settings.py
+
+If you like, you can add this command to your ``~/.bashrc`` to have it
+executed every time you open a shell.
+
+Another way is by prefixing your invocations with
+``MUTALYZER_SETTINGS=...``. For example::
+
+ $ MUTALYZER_SETTINGS=~/mutalyzer/settings.py mutalyzer-website
+
+
+Example configuration
+---------------------
+
+If you followed the steps in :ref:`install`, this is a standard configuration
+file that will work for you:
+
+.. code-block:: python
+
+ REDIS_URI = 'redis://localhost'
+ DATABASE_URI = 'postgresql://mutalyzer:*****@localhost/mutalyzer'
+
+This is not yet a minimal configuration. In fact, you can run Mutalyzer
+without a configuration file since the default configuration works out of the
+box. The default configuration uses a mock Redis instance, an in-memory
+database backend and a temporary directory for cache and log files, so it is
+not recommended for anything more than playing around.
+
+The next section describes all available configuration settings.
+
+
+Configuration settings
+----------------------
+
+.. note:: Todo: This section is incomplete.
+
+Note that the configuration file is interpreted as a Python module, so you can
+use arbitrary Python expressions as configuration values, or even import other
+modules in it.
+
+Unsetting a configuration setting is done by using the value `None`. If no
+default value is mentioned for any configuration setting below it means it is
+not set by default.
+
+
+Database settings
+^^^^^^^^^^^^^^^^^
+
+DATABASE_URI
+ SQLAlchemy database connection URI specifying the database used to store
+ users, samples, variants, etcetera.
+
+ ================ =====================================================
+ Database system Example URI
+ ================ =====================================================
+ PostgreSQL ``postgresql://mutalyzer:*****@localhost/mutalyzer``
+ MySQL ``mysql://mutalyzer:*****@localhost/mutalyzer``
+ SQLite ``sqlite:////tmp/mutalyzer.db``
+ ================ =====================================================
+
+ See the SQLAlchemy documentation on
+ `Engine Configuration
+ <http://docs.sqlalchemy.org/en/latest/core/engines.html>`_ for more
+ information.
+
+ `Default value:` ``sqlite://`` (in-memory SQLite database)
+
+
+Miscellaneous settings
+^^^^^^^^^^^^^^^^^^^^^^
+
+TESTING
+ If set to `True`, Mutalyzer assumes to be running its unit tests. This is
+ done automatically in the provided test suite, so you should never have to
+ change this setting.
+
+ `Default value:` `False`
diff --git a/doc/copyright.rst b/doc/copyright.rst
new file mode 100644
index 0000000000000000000000000000000000000000..aaa61508e29da904ad597035f59735278b25b954
--- /dev/null
+++ b/doc/copyright.rst
@@ -0,0 +1,10 @@
+Copyright
+=========
+
+.. include:: ../LICENSE.rst
+
+
+Authors
+-------
+
+.. include:: ../AUTHORS.rst
diff --git a/doc/deploy.rst b/doc/deploy.rst
new file mode 100644
index 0000000000000000000000000000000000000000..8779aa6fdda223f45eed47a65abd83a0383b422a
--- /dev/null
+++ b/doc/deploy.rst
@@ -0,0 +1,175 @@
+.. highlight:: none
+
+.. _deploy:
+
+Deploying Mutalyzer in production
+=================================
+
+The previous sections discussed managing a Mutalyzer installation with a focus
+on a development environment. There are a number of additional things you will
+want to consider when deploying Mutalyzer in a production environment, mainly
+concerning security and performance.
+
+Usually you'll at least want to use a well-performing WSGI application server
+for the website and SOAP and HTTP/RPC+JSON webservices. There are many options
+here, ranging from Apache's `mod_wsgi`_ to `uWSGI`_ to standalone WSGI
+containers such as `Gunicorn`_.
+
+Below we briefly describe our recommended setup for a production environment
+using Gunicorn, nginx and Supervisor.
+
+
+Configuration settings
+----------------------
+
+Todo: Link to the description of these configuration settings.
+
+It is recommended to at least set the following configuration settings:
+
+- DEBUG
+- EMAIL
+- CACHE_DIR
+- SOAP_WSDL_URL
+- JSON_ROOT_URL
+
+
+WSGI application server: Gunicorn
+---------------------------------
+
+`Gunicorn`_ is a well-perfoming Python WSGI HTTP Server. Being a Python
+application, it can be installed in the Mutalyzer virtual environment with
+``pip install gunicorn``.
+
+Many configuration settings are available for Gunicorn and we recommend to use
+a configuration file per WSGI application. For example, the following
+configuration can be stored in ``website.conf``:
+
+.. code-block:: ini
+
+ workers = 4
+ max_requests = 1000
+ timeout = 600
+ bind = 'unix:/opt/mutalyzer/run/website.sock'
+
+This will bind the Gunicorn server to a unix socket (which we can later use
+from nginx) and run with 4 worker processes. To serve the Mutalyzer website
+with this configuration, run the following::
+
+ $ gunicorn -c website.conf mutalyzer.entrypoints.website
+
+This uses the WSGI application object exported by the
+`mutalyzer.entrypoints.website` module. Likewise, the SOAP and HTTP/RPC+JSON
+webservices have WSGI application objects exported by the
+`mutalyzer.entrypoints.service_soap` and `mutalyzer.entrypoints.service_json`
+modules.
+
+
+Web server: nginx
+-----------------
+
+It is usually a good idea to use a separate webserver in front of the WSGI
+application servers. We use `nginx`_ for this purpose and configure it to
+server static files directly and act as a reverse proxy for the WSGI
+applications.
+
+For example, to serve the website from the root path and the HTTP/RPC+JSON
+webservice from the ``/json`` path, an nginx configuration similar to the
+following can be used:
+
+.. code-block:: nginx
+
+ server {
+ listen 80;
+ server_name _;
+
+ client_max_body_size 2G;
+ keepalive_timeout 5;
+
+ location /static/ {
+ alias /opt/mutalyzer/static/;
+ expires 30d;
+ add_header Pragma public;
+ add_header Cache-Control "public";
+ }
+
+ location / {
+ root /usr/share/nginx/html;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Scheme $scheme;
+ proxy_set_header Host $http_host;
+ proxy_redirect off;
+ proxy_read_timeout 600;
+ proxy_pass http://website;
+ }
+
+ location /json {
+ root /usr/share/nginx/html;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Scheme $scheme;
+ proxy_set_header Host $http_host;
+ proxy_redirect off;
+ proxy_read_timeout 600;
+ proxy_pass http://service-json;
+ }
+ }
+
+ upstream website {
+ server unix:/opt/mutalyzer/run/website.sock fail_timeout=0;
+ }
+
+ upstream service-json {
+ server unix:/opt/mutalyzer/run/service-json.sock fail_timeout=0;
+ }
+
+
+Process control: Supervisor
+---------------------------
+
+For managing the different WSGI application servers and Mutalyzer batch
+processor, Supervisor can be used. Supervisor is usually started from the init
+system and controls programs and program groups. For example, it can
+automatically restart a program if it crashed for some reason.
+
+The following is an example Supervisor configuration defining a Mutalyzer
+group consisting of the batch processor and a Gunicorn process for the
+website:
+
+.. code-block:: ini
+
+ [group:mutalyzer]
+ programs=batch-processor,website
+
+ [program:batch-processor]
+ command=mutalyzer-batch-processor
+ autorestart=true
+ environment=MUTALYZER_SETTINGS="/opt/mutalyzer/conf/settings.py"
+
+ [program:website]
+ command=gunicorn -c /opt/mutalyzer/conf/website.conf mutalyzer.entrypoints.website
+ autorestart=true
+ environment=MUTALYZER_SETTINGS="/opt/mutalyzer/conf/settings.py"
+
+
+Automated deployment with Ansible
+---------------------------------
+
+Deployments of complete production environments are often complex and
+repetitive. Therefore, manual deployments are inefficient and
+error-prone. Several systems exist to automate this, such as `Puppet`_,
+`Chef`_, and `Ansible`_.
+
+An automated deployment of Mutalyzer with Ansible is `available from the LUMC
+GitLab <https://git.lumc.nl/mutalyzer/mutalyzer-deployment>`_. This includes
+installation of the website, SOAP and HTTP/RPC+JSON webservices, and the batch
+processor, similar to the setup described above.
+
+
+.. _Ansible: http://www.ansible.com/
+.. _Chef: http://www.getchef.com/
+.. _Gunicorn: http://gunicorn.org/
+.. _mod_wsgi: https://code.google.com/p/modwsgi/
+.. _nginx: http://nginx.org/
+.. _Puppet: http://puppetlabs.com/
+.. _uWSGI: http://uwsgi-docs.readthedocs.org/
diff --git a/doc/design.rst b/doc/design.rst
new file mode 100644
index 0000000000000000000000000000000000000000..f02263126171c4d753760725d36ba9c6e19868df
--- /dev/null
+++ b/doc/design.rst
@@ -0,0 +1,4 @@
+Application design
+==================
+
+Todo: Write this section and integrate the LaTeX technical reference.
diff --git a/doc/development.rst b/doc/development.rst
new file mode 100644
index 0000000000000000000000000000000000000000..9bf0b9bb5b61c0d3c1b53fdf6e77fb9265957eda
--- /dev/null
+++ b/doc/development.rst
@@ -0,0 +1,133 @@
+.. highlight:: none
+
+.. _development:
+
+Development
+===========
+
+Development of Mutalyzer happens on GitLab:
+https://git.lumc.nl/mutalyzer/mutalyzer
+
+
+Contributing
+------------
+
+Contributions to Mutalyzer are very welcome! They can be feature requests, bug
+reports, bug fixes, unit tests, documentation updates, or anything els you may
+come up with.
+
+
+Coding style
+------------
+
+In general, try to follow the `PEP 8`_ guidelines for Python code and `PEP
+257`_ for docstrings.
+
+
+Unit tests
+----------
+
+To run the unit tests with `nose`_, just run ``nosetests -v``.
+
+
+Working with feature branches
+-----------------------------
+
+New features are best implemented in their own branches, isolating the work
+from unrelated developments. In fact, it's good practice to *never work
+directly on the master branch* but always in a separate branch. For this
+reason, the master branch on the GitLab server is locked. Feature branches can
+be merged back into master via a *merge request* in GitLab.
+
+Before starting work on your feature, create a branch for it::
+
+ git checkout -b your-feature
+
+Commit changes on this branch. If you're happy with it, push to GitLab::
+
+ git push origin your-feature -u
+
+Now create a merge request to discuss the implementation with your
+colleagues. This might involve adding additional commits which are included in
+the merge request by pushing your branch again::
+
+ git commit
+ git push
+
+You may also be asked to rebase your branch on the master branch if it has
+changed since you started your work. This will require a forced push::
+
+ git fetch
+ git rebase origin/master
+ git push -f
+
+If the work is done, a developer can merge your branch and close the merge
+request. After the branch was merged you can safely delete it::
+
+ git branch -d your-feature
+
+
+Versioning
+----------
+
+All version numbers for recent Mutalyzer releases take the form 2.0.beta-X
+where X is incremented on release. Pre-release (or development) version
+numbers take the form 2.0.beta-X.dev where 2.0.beta-X is the closest future
+release version.
+
+Note that we are planning a switch to `SemVer`_.
+
+.. A normal version number takes the form X.Y.Z where X is the major version, Y
+ is the minor version, and Z is the patch version. Development versions take
+ the form X.Y.Z.dev where X.Y.Z is the closest future release version.
+
+ Note that this scheme is not 100% compatible with `SemVer`_ which would
+ require X.Y.Z-dev instead of X.Y.Z.dev but `compatibility with setuptools
+ <http://peak.telecommunity.com/DevCenter/setuptools#specifying-your-project-s-version>`_
+ is more important for us. Other than that, version semantics are as described
+ by SemVer.
+
+ Releases are `published at PyPI <https://pypi.python.org/pypi/wiggelen>`_ and
+ available from the GitHub git repository as tags.
+
+
+Release procedure
+^^^^^^^^^^^^^^^^^
+
+Releasing a new version is done as follows:
+
+1. Make sure the section in the ``CHANGES`` file for this release is
+ complete and there are no uncommitted changes.
+
+ .. note::
+
+ Commits since release 2.0.beta-X can be listed with ``git log
+ mutalyzer-2.0.beta-X..`` for quick inspection.
+
+2. Update the ``CHANGES`` file to state the current date for this release
+ and edit ``mutalyzer/__init__.py`` by updating `__date__`, removing the
+ ``dev`` value from `__version_info__` and setting `RELEASE` to `True`.
+
+ Commit and tag the version update::
+
+ git commit -am 'Bump version to 2.0.beta-X'
+ git tag -a 'mutalyzer-2.0.beta-X'
+ git push --tags
+
+3. Add a new entry at the top of the ``CHANGES`` file like this::
+
+ Version 2.0.beta-Y
+ ------------------
+
+ Release date to be decided.
+
+ Set `__version_info__` to a new version ending with ``dev`` and set
+ `RELEASE` to `True` in ``mutalyzer/__init__.py``. Commit these changes::
+
+ git commit -am 'Open development for 2.0.beta-Y'
+
+
+.. _nose: https://nose.readthedocs.org/
+.. _PEP 8: http://www.python.org/dev/peps/pep-0008/
+.. _PEP 257: http://www.python.org/dev/peps/pep-0257/
+.. _SemVer: http://semver.org/
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..60c51aa5c3cc3313b0411e10fe26565497d8bb79
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,56 @@
+Mutalyzer developer documentation
+=================================
+
+Mutalyzer is an HGVS variant nomenclature checker. The canonical Mutalyzer
+installation can be found at [mutalyzer.nl](https://mutalyzer.nl).
+
+This is the developer documentation for Mutalyzer. User documentation can be
+found on the `wiki <https://humgenprojects.lumc.nl/trac/mutalyzer>`_.
+
+
+Managing Mutalyzer
+------------------
+
+Information for getting Mutalyzer running on a system.
+
+.. toctree::
+ :maxdepth: 2
+
+ install
+ config
+ run
+ upgrade
+ admin
+ deploy
+
+
+Technical reference
+-------------------
+
+Application design and complete API reference.
+
+.. toctree::
+ :maxdepth: 2
+
+ design
+ api
+
+
+Additional notes
+----------------
+
+.. toctree::
+ :maxdepth: 2
+
+ development
+ todo
+ changelog
+ copyright
+
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/doc/install.rst b/doc/install.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c23959e09d89cafd9ce26b120739ef183b7b71f9
--- /dev/null
+++ b/doc/install.rst
@@ -0,0 +1,218 @@
+.. highlight:: none
+
+.. _install:
+
+Installation
+============
+
+Mutalyzer depends on a database server, `Python`_ 2.7, and several Python
+packages. `Redis`_ is a soft dependency. This section walks you through
+installing Mutalyzer with Redis and using `PostgreSQL`_ as database server,
+which is the recommended setup.
+
+.. note:: All operating system specific instructions assume installation on a
+ `Debian`_ 7 *wheezy* system. You'll have to figure out the necessary
+ adjustements yourself if you're on another system.
+
+The following steps will get Mutalyzer running on your system with the
+recommended setup:
+
+* :ref:`install-postgresql`
+* :ref:`install-redis`
+* :ref:`install-virtualenv`
+* :ref:`install-setup`
+
+At the bottom of this page some :ref:`alternative setups
+<install-alternatives>` are documented.
+
+
+.. _install-quick:
+
+If you're in a hurry
+--------------------
+
+The impatient can run Mutalyzer without a database server and more such
+nonsense with the following steps::
+
+ $ pip install -r requirements.txt
+ $ MUTALYZER_SETTINGS=/dev/null python -m mutalyzer.entrypoints.website
+
+This starts the website frontend on the reported port using an in-memory
+SQLite database.
+
+
+.. _install-postgresql:
+
+Database server: PostgreSQL
+---------------------------
+
+Install `PostgreSQL`_ and add a user for Mutalyzer. Create a database
+(e.g. ``mutalyzer``) owned by the new user. For example::
+
+ $ sudo apt-get install postgresql
+ $ sudo -u postgres createuser --superuser $USER
+ $ createuser --pwprompt --encrypted --no-adduser --no-createdb --no-createrole mutalyzer
+ $ createdb --encoding=UNICODE --owner=mutalyzer mutalyzer
+
+Also install some development libraries needed for building the ``psycopg2``
+Python package later and add the package to the list of requirements::
+
+ $ sudo apt-get install libpq-dev
+ $ echo psycopg2 >> requirements.txt
+
+This will make sure the Python PostgreSQL database adapter gets installed in
+the :ref:`install-virtualenv` section.
+
+.. seealso::
+
+ :ref:`install-mysql`
+ Alternatively, MySQL can be used as database server.
+
+ :ref:`install-sqlite`
+ Alternatively, SQLite can be used as database server.
+
+ `Dialects -- SQLAlchemy documentation <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`_
+ In theory, any database supported by SQLAlchemy could work.
+
+
+.. _install-redis:
+
+Redis
+-----
+
+Mutalyzer uses Redis for non-critical fast storage such as statistics::
+
+ $ sudo apt-get install redis-server
+
+.. note:: Redis is a soft dependency, meaning that Mutalyzer will run without
+ it (but may lack some non-essential features).
+
+
+.. _install-virtualenv:
+
+Python virtual environment
+--------------------------
+
+It is recommended to run Mutalyzer from a Python virtual environment, using
+`virtualenv`_. Installing virtualenv and creating virtual environments is not
+covered here.
+
+Assuming you created and activated a virtual environment for Mutalyzer,
+install all required Python packages::
+
+ $ sudo apt-get install python-dev libmysqlclient-dev
+ $ pip install -r requirements.txt
+
+Now might be a good time to run the unit tests::
+
+ $ nosetests -v
+
+If everything's okay, install Mutalyzer::
+
+ $ python setup.py install
+
+.. seealso::
+
+ `virtualenv`_
+ ``virtualenv`` is a tool to create isolated Python environments.
+
+ `virtualenvwrapper`_
+ ``virtualenvwrapper`` is a set of extensions to the ``virtualenv``
+ tool. The extensions include wrappers for creating and deleting virtual
+ environments and otherwise managing your development workflow.
+
+
+.. _install-setup:
+
+Mutalyzer setup
+---------------
+
+Mutalyzer looks for its configuration in the file specified by the
+``MUTALYZER_SETTINGS`` environment variable. First create the file with your
+configuration settings, for example::
+
+ $ export MUTALYZER_SETTINGS=~/mutalyzer/settings.py
+ $ cat > $MUTALYZER_SETTINGS
+ REDIS_URI = 'redis://localhost'
+ DATABASE_URI = 'postgresql://mutalyzer:*****@localhost/mutalyzer'
+
+A script is included to setup the database::
+
+ $ mutalyzer-admin setup-database --alembic-config migrations/alembic.ini
+
+You can now proceed to :ref:`run`.
+
+.. seealso::
+
+ :ref:`config`
+ For more information on the available configuration settings.
+
+
+.. _install-alternatives:
+
+Alternative setups
+------------------
+
+The remainder of this page documents some alternatives to the recommended
+setup documented above.
+
+
+.. _install-mysql:
+
+Database server: MySQL
+^^^^^^^^^^^^^^^^^^^^^^
+
+Install `MySQL`_ and create a database (e.g. ``mutalyzer``) with all privileges
+for the Mutalyzer user. For example::
+
+ $ sudo apt-get install mysql-server
+ $ mysql -h localhost -u root -p
+ > create database mutalyzer;
+ > grant all privileges on mutalyzer.* to mutalyzer@localhost identified by '*****';
+
+The Python MySQL database adapter is a hard dependency regardless of your
+choice of database server, so it'll get installed in the
+:ref:`install-virtualenv` section.
+
+In the :ref:`install-setup` section, make sure to use a MySQL database URI in
+the Mutalyzer settings file, e.g.:
+
+.. code-block:: python
+
+ DATABASE_URI = 'mysql://mutalyzer:*****@localhost/mutalyzer'
+
+.. seealso::
+
+ :ref:`install-postgresql`
+ The recommended setup uses PostgreSQL as database server.
+
+
+.. _install-sqlite:
+
+Database server: SQLite
+^^^^^^^^^^^^^^^^^^^^^^^
+
+You probably already have all you need for using `SQLite`_, so this section
+consists of zero steps.
+
+Just note that in the :ref:`install-setup` section, you should use an SQLite
+database URI in the Mutalyzer settings file, e.g.:
+
+.. code-block:: python
+
+ DATABASE_URI = 'sqlite:////tmp/mutalyzer.db'
+
+.. seealso::
+
+ :ref:`install-postgresql`
+ The recommended setup uses PostgreSQL as database server.
+
+
+.. _Debian: http://www.debian.org/
+.. _MySQL: http://www.mysql.com/
+.. _PostgreSQL: http://www.postgresql.org/
+.. _Python: http://python.org/
+.. _Redis: http://redis.io/
+.. _SQLite: http://www.sqlite.org/
+.. _virtualenv: http://www.virtualenv.org/
+.. _virtualenvwrapper: http://www.doughellmann.com/docs/virtualenvwrapper/
diff --git a/doc/run.rst b/doc/run.rst
new file mode 100644
index 0000000000000000000000000000000000000000..269ba126463dc50ea060c1ef3bd72233183d8fbc
--- /dev/null
+++ b/doc/run.rst
@@ -0,0 +1,32 @@
+.. highlight:: none
+
+.. _run:
+
+Running Mutalyzer
+=================
+
+Mutalyzer comes with a number of different interfaces, of which the website is
+perhaps the main one. It can be started using a built-in test server that's
+useful for development and debugging purposes like this::
+
+ $ mutalyzer-website
+ * Running on http://127.0.0.1:5000/
+
+You can now point your webbrowser to the URL that is printed and see the
+welcoming Mutalyzer homepage.
+
+Likewise, the SOAP and HTTP/RPC+JSON webservices can be started with the
+``mutalyzer-service-json`` and ``mutalyzer-service-soap`` commands,
+respectively.
+
+For processing batch jobs, the batch processor must be running. This process
+can be started from the command line and will keep running until it is stopped
+by pressing Ctrl+C::
+
+ $ mutalyzer-batch-processor
+ ^Cmutalyzer-batch-processor: Hitting Ctrl+C again will terminate any running job!
+ mutalyzer-batch-processor: Graceful shutdown
+
+The built-in test servers won't get you far in production, though, and there
+are many other possibilities for deploying Mutalyzer using WSGI. This topic is
+discussed in :ref:`deploy`.
diff --git a/doc/todo.rst b/doc/todo.rst
new file mode 100644
index 0000000000000000000000000000000000000000..00e5fbf86e828784e53866e1313594c0d6200ae6
--- /dev/null
+++ b/doc/todo.rst
@@ -0,0 +1,53 @@
+Todo list
+=========
+
+These are some general todo notes. More specific notes can be found by
+grepping the source code for ``Todo``.
+
+.. seealso::
+
+ `Mutalyzer Trac -- Active tickets <https://humgenprojects.lumc.nl/trac/mutalyzer/report/2>`_
+ Users can file tickets on the Mutalyzer Trac website.
+
+ `Mutalyzer GitLab -- Open issues <https://git.lumc.nl/mutalyzer/mutalyzer/issues>`_
+ Some issues are recorded in the Mutalyzer GitLab project.
+
+- Improve the web interface design :)
+- Test all uses of mkstemp().
+- Use naming conventions for modules Crossmap, Db, File, GenRecord, Retriever
+ and Scheduler.
+- Use standard logging module, with rotating functionality. Race conditions
+ on the log file are probably a problem in the current setup.
+ Instead of that rotating, we could also use logrotate:
+ http://serverfault.com/questions/55610/logrotate-and-open-files
+- Setup continuous integration. Currently, I'm most impressed with Hudson.
+ - http://hudson-ci.org/
+ - http://www.rhonabwy.com/wp/2009/11/04/setting-up-a-python-ci-server-with-hudson/
+ Or perhaps Jenkins.
+ - http://jenkins-ci.org/
+- Migrate Javascript to JQuery.
+- I think in the long run, the Output object is not really the way to go. It
+ obscures the control flow. The logging part should use the standard logging
+ module. The data gathering by the Output object is probably better handled
+ by explicitely returning data objects from functions.
+- Migrate from TAL to a more mondern and maintained Python template library,
+ for example jinja.
+- Develop a large test suite.
+- Create a web interface url to watch the progress of a batch job.
+- Create web services for the batch jobs (steal ideas from Jeroen's DVD
+ web service).
+- Use virtualenv?
+- Use SQLAlchemy?
+- Password for MySQL user.
+- In deployment, remove old versions of Mutalyzer package?
+- Check for os.path.join vulnerabilities.
+- Use a standard solution for the database migrations in extras/migrations.
+- Use something like Sphinx to generate development documentation from code.
+- There are some problems with the batch architecture, especially that there
+ cannot be multiple workers without synchronisation problems.
+ Good read: http://news.ycombinator.com/item?id=3002861
+ Suggestion: http://celeryproject.org/
+- Have a normal 404 page.
+- Maintenance (and/or read-only) mode.
+- Cleanup this document.
+- Be more explicit in all the type of descriptions we don't currently support.
diff --git a/doc/upgrade.rst b/doc/upgrade.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a9bd4697537beeea4b093b419c90483171b0ac29
--- /dev/null
+++ b/doc/upgrade.rst
@@ -0,0 +1,26 @@
+.. highlight:: none
+
+.. _upgrade:
+
+Upgrading
+=========
+
+Before upgrading Mutalyzer, stop any currently running instances. Then, update
+your copy of the source code (using for example ``git pull`` on an existing
+git clone).
+
+Make sure to install any new requirements::
+
+ $ pip install -r requirements.txt
+
+Now install the new version::
+
+ $ python setup.py install
+
+Managing database migrations is done using `Alembic`_. This command will move
+your database to the latest schema::
+
+ $ alembic -c migrations/alembic.ini upgrade head
+
+
+.. _Alembic: http://alembic.readthedocs.org/
diff --git a/extras/apache/mutalyzer.conf b/extras/apache/mutalyzer.conf
deleted file mode 100644
index 4149ef44ede75559607b92519c813e65c503047a..0000000000000000000000000000000000000000
--- a/extras/apache/mutalyzer.conf
+++ /dev/null
@@ -1,36 +0,0 @@
-# Static files
-Alias /mutalyzer/static /var/www/mutalyzer/static
-<Directory /var/www/mutalyzer/static>
- Order deny,allow
- Allow from all
- Options -Indexes
- AllowOverride None
-</Directory>
-
-# Use daemon mode of mod_wsgi
-WSGIDaemonProcess mutalyzer processes=2 threads=15 maximum-requests=10000
-WSGIProcessGroup mutalyzer
-
-# SOAP/1.1 web service
-WSGIScriptAlias /mutalyzer/services <MUTALYZER_BIN_SOAP_SERVICE>
-<Directory /mutalyzer/services>
- Order deny,allow
- Allow from all
- Options -Indexes
-</Directory>
-
-# HTTP/RPC+JSON web service
-WSGIScriptAlias /mutalyzer/json <MUTALYZER_BIN_JSON_SERVICE>
-<Directory /mutalyzer/json>
- Order deny,allow
- Allow from all
- Options -Indexes
-</Directory>
-
-# Website
-WSGIScriptAlias /mutalyzer <MUTALYZER_BIN_WEBSITE>
-<Directory /mutalyzer>
- Order deny,allow
- Allow from all
- Options -Indexes
-</Directory>
diff --git a/extras/config.example b/extras/config.example
deleted file mode 100644
index 251ca14333ac6cf18b68d62ccb632fc47947aa1c..0000000000000000000000000000000000000000
--- a/extras/config.example
+++ /dev/null
@@ -1,173 +0,0 @@
-#
-# Mutalyzer config file.
-#
-# Specify the location of this file in the MUTALYZER_SETTINGS environment
-# variable.
-
-
-#
-# These settings are used by the Retriever module.
-#
-
-# Use this email address for retrieval of records at the NCBI.
-email = "mutalyzer@humgen.nl"
-
-# The cache directory.
-cache = "/var/cache/mutalyzer"
-
-# The maximum size of the cache in megabytes.
-cachesize = 50
-
-# The maximum size of a downloaded GenBank file in megabytes.
-maxDldSize = 10
-
-# The minimum size of a downloaded GenBank file in bytes.
-minDldSize = 512
-
-# The URL from where LRG files are fetched
-lrgurl = "ftp://ftp.ebi.ac.uk/pub/databases/lrgex/"
-
-
-#
-# These settings are used by the Db module.
-#
-
-# Internal database.
-internalDb = "mutalyzer"
-
-# MySQL mapping database names.
-dbNames = "hg18", "hg19", "mm10"
-
-# Default mapping database.
-defaultDb = "hg19"
-
-# MySQL username for the local databases (internalDb and dbNames).
-LocalMySQLuser = "mutalyzer"
-
-# Host name for the local databases.
-LocalMySQLhost = "localhost"
-
-# Automatically reconnect to MySQL server using the MySQLdb reconnect option.
-# Note that this may not always be available and if not will result in an
-# error if used. Mutalyzer also implements its own reconnecting mechanism now
-# which is always on. See Trac issue #91.
-autoReconnect = no
-
-# Number of days a cached transcript->protein link from the NCBI is valid.
-proteinLinkLifetime = 30
-
-# Number of days a cached nonexisting transcript->protein link from the NCBI
-# is valid.
-proteinLinkNoneLifetime = 5
-
-
-#
-# These settings are used by the Output module.
-#
-
-# Name and location of the log file.
-log = "/var/log/mutalyzer.log"
-
-# Prefix for each log message.
-datestring = "%Y-%m-%d %H:%M:%S"
-
-# Message levels:
-#
-# 0 : Debug ; Show all messages.
-# 1 : Info ; Show all messages except debug messages.
-# 2 : Warning ; Show warning, error and fatal messages.
-# 3 : Error ; Show error and fatal messages.
-# 4 : Fatal ; Only show fatal messages.
-# 5 : Off ; Show nothing.
-
-# Level of logged messages.
-loglevel = 3
-
-# Level of output messages.
-outputlevel = 1
-
-# Show debug info in the web interface.
-debug = yes
-
-
-#
-# These settings are used by the Mutator module.
-#
-
-# Length of the flanking sequences (used in the visualisation of mutations).
-flanksize = 25
-
-# Maximum length of visualised mutations.
-maxvissize = 25
-
-# Length of the flanking sequences of the clipped mutations (see maxvissize).
-flankclipsize = 6
-
-
-#
-# These settings are used by the Scheduler module.
-#
-
-# Return e-mail address.
-mailFrom = "noreply@humgen.nl"
-
-# Subject of the message.
-mailSubject = "Result of Mutalyzer batch check."
-
-# Location of the results.
-resultsDir = "/var/cache/mutalyzer"
-
-# Location of the PID file.
-PIDfile = "/var/run/mutalyzer/mutalyzer-batchd.pid"
-
-# Maximum size for uploaded batch input files in megabytes.
-batchInputMaxSize = 5
-
-# The output header for NameChecking
-nameCheckOutHeader = "Input", "Errors | Messages", "AccNo", "Genesymbol", "Variant", "Reference Sequence Start Descr.", "Coding DNA Descr.", "Protein Descr.", "GeneSymbol Coding DNA Descr.", "GeneSymbol Protein Descr.", "Genomic Reference", "Coding Reference", "Protein Reference", "Affected Transcripts", "Affected Proteins", "Restriction Sites Created", "Restriction Sites Deleted"
-
-# The output header for SyntaxChecking
-syntaxCheckOutHeader = "Input", "Status"
-
-# The output header for PositionConverter
-positionConverterOutHeader = "Input Variant", "Errors", "Chromosomal Variant", "Coding Variant(s)"
-
-# The output header for SnpConverter
-snpConverterOutHeader = "Input Variant", "HGVS description(s)", "Errors | Messages"
-
-
-#
-# These settings are used by the File module.
-#
-
-# Amount of bytes to be read for determining the file type.
-bufSize = 32768
-
-# The obligatory header in batch request files.
-header = "AccNo", "Genesymbol", "Mutation"
-
-# Threshold for Batch Jobs
-threshold = 0.05
-
-
-#
-# These settings are used by the GenRecord module.
-#
-
-spliceAlarm = 2
-
-spliceWarn = 5
-
-
-#
-# These settings are for Piwik analytics.
-#
-
-# Enable Piwik analytics.
-piwik = no
-
-# Piwik server base URL (include protocol, no trailing slash).
-piwikBase = https://piwik.example.com
-
-# Piwik site ID.
-piwikSite = 1
diff --git a/extras/config.user.example b/extras/config.user.example
deleted file mode 100644
index fd3f1428a8ee37d248bed33a19316948cb342a1d..0000000000000000000000000000000000000000
--- a/extras/config.user.example
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# Mutalyzer config file.
-#
-# Specify the location of this file in the MUTALYZER_SETTINGS environment
-# variable.
-
-# Use this email address for retrieval of records at the NCBI.
-email = "mutalyzer@humgen.nl"
-
-# The cache directory.
-cache = "/home/<USERNAME>/.cache/mutalyzer"
-
-# Name and location of the log file.
-log = "/tmp/mutalyzer-<USERNAME>.log"
diff --git a/extras/cron.d/mutalyzer-cache-sync b/extras/cron.d/mutalyzer-cache-sync
deleted file mode 100644
index 02d3e0a349e261bdae898929d7629a768fdde4f5..0000000000000000000000000000000000000000
--- a/extras/cron.d/mutalyzer-cache-sync
+++ /dev/null
@@ -1,2 +0,0 @@
-# Synchronize the local cache with the live server every morning at 05:25
-#25 5 * * * www-data <MUTALYZER_BIN_CACHE_SYNC> 'https://mutalyzer.nl/services/?wsdl' 'https://mutalyzer.nl/Reference/{file}' 3
diff --git a/extras/cron.d/mutalyzer-mapping-update b/extras/cron.d/mutalyzer-mapping-update
deleted file mode 100644
index c4648431866d4eec3687d69605d7ff49d59010a7..0000000000000000000000000000000000000000
--- a/extras/cron.d/mutalyzer-mapping-update
+++ /dev/null
@@ -1,6 +0,0 @@
-# Update the mapping database every sunday morning at 03:25 and 04:25
-#25 3 * * 7 www-data wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.36.3/mapview/seq_gene.md.gz" -O - | zcat > /tmp/seq_gene.md; <MUTALYZER_BIN_MAPPING_UPDATE> hg18 /tmp/seq_gene.md reference
-#24 4 * * 7 www-data wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.37.1/mapview/seq_gene.md.gz" -O - | zcat > /tmp/seq_gene.md; <MUTALYZER_BIN_MAPPING_UPDATE> hg19 /tmp/seq_gene.md 'Primary Assembly'
-#25 4 * * 7 www-data wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.37.2/mapview/seq_gene.md.gz" -O - | zcat > /tmp/seq_gene.md; <MUTALYZER_BIN_MAPPING_UPDATE> hg19 /tmp/seq_gene.md 'GRCh37.p2-Primary Assembly'
-#25 4 * * 7 www-data wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.37.3/mapview/seq_gene.md.gz" -O - | zcat > /tmp/seq_gene.md; <MUTALYZER_BIN_MAPPING_UPDATE> hg19 /tmp/seq_gene.md 'GRCh37.p5-Primary Assembly'
-#25 4 * * 7 www-data wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/mapview/seq_gene.md.gz" -O - | zcat > /tmp/seq_gene.md; <MUTALYZER_BIN_MAPPING_UPDATE> hg19 /tmp/seq_gene.md 'GRCh37.p10-Primary Assembly'
diff --git a/extras/init.d/mutalyzer-batchd b/extras/init.d/mutalyzer-batchd
deleted file mode 100644
index ea14453ad5cd2a59cb00f3d2af4e13296cf1b062..0000000000000000000000000000000000000000
--- a/extras/init.d/mutalyzer-batchd
+++ /dev/null
@@ -1,132 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides: mutalyzer-batchd
-# Required-Start: $local_fs $remote_fs $network $syslog $mysql
-# Required-Stop: $local_fs $remote_fs $network $syslog $mysql
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: Start and stop the Mutalyzer batch daemon
-# Description: Controls the Mutalyzer batch job processing daemon..
-### END INIT INFO
-
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="Mutalyzer batch deamon"
-NAME=mutalyzer-batchd
-DAEMON=<MUTALYZER_BIN_BATCHD>
-DIR=/
-PIDDIR=/var/run/mutalyzer
-PIDFILE=$PIDDIR/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-USER=www-data
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
-
-# Load the VERBOSE setting and other rcS variables
-. /lib/init/vars.sh
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
-. /lib/lsb/init-functions
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
- # Return
- # 0 if daemon has been started
- # 1 if daemon was already running
- # 2 if daemon could not be started
- mkdir -p $PIDDIR
- chown -R $USER $PIDDIR
- start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --chuid $USER --chdir $DIR --test > /dev/null \
- || return 1
- start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --chuid $USER --chdir $DIR -- \
- $DAEMON_ARGS \
- || return 2
- # Add code here, if necessary, that waits for the process to be ready
- # to handle requests from services started subsequently which depend
- # on this one. As a last resort, sleep for some time.
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
- # Return
- # 0 if daemon has been stopped
- # 1 if daemon was already stopped
- # 2 if daemon could not be stopped
- # other if a failure occurred
- start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
- RETVAL="$?"
- [ "$RETVAL" = 2 ] && return 2
- # Many daemons don't delete their pidfiles when they exit.
- rm -f $PIDFILE
- return "$RETVAL"
-}
-
-case "$1" in
- start)
- [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
- do_start
- case "$?" in
- 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
- 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
- esac
- ;;
- stop)
- [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
- do_stop
- case "$?" in
- 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
- 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
- esac
- ;;
- status)
- status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
- ;;
- #reload|force-reload)
- #
- # If do_reload() is not implemented then leave this commented out
- # and leave 'force-reload' as an alias for 'restart'.
- #
- #log_daemon_msg "Reloading $DESC" "$NAME"
- #do_reload
- #log_end_msg $?
- #;;
- restart|force-reload)
- #
- # If the "reload" option is implemented then remove the
- # 'force-reload' alias
- #
- log_daemon_msg "Restarting $DESC" "$NAME"
- do_stop
- case "$?" in
- 0|1)
- do_start
- case "$?" in
- 0) log_end_msg 0 ;;
- 1) log_end_msg 1 ;; # Old process is still running
- *) log_end_msg 1 ;; # Failed to start
- esac
- ;;
- *)
- # Failed to stop
- log_end_msg 1
- ;;
- esac
- ;;
- *)
- #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2
- echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
- exit 3
- ;;
-esac
-
-:
diff --git a/extras/migrations/001-db-gbinfo-add-created.migration b/extras/migrations/001-db-gbinfo-add-created.migration
deleted file mode 100755
index 686563ce39af39dddd5fe1e66afcab9e10a69721..0000000000000000000000000000000000000000
--- a/extras/migrations/001-db-gbinfo-add-created.migration
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-Add a column and index 'created' to the 'GBInfo' table.
-
-Usage:
- ./001-db-gbinfo-add-created.migration [migrate]
-"""
-
-
-import migration
-
-
-def check():
- """
- Check if migration is needed.
- """
- connection = migration.db_connect('mutalyzer')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM GBInfo WHERE field = "created";')
- has_column = len(cursor.fetchall()) > 0
- cursor.execute('SHOW INDEX FROM GBInfo WHERE Key_name = "created";')
- has_index = len(cursor.fetchall()) > 0
- connection.close()
- if has_column != has_index:
- migration.fatal('Installation is not in a recognizable state. Fix manually.')
- return not has_column
-
-
-def migrate():
- """
- Perform migration.
- """
- connection = migration.db_connect('mutalyzer')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE GBInfo
- ADD COLUMN created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- ADD INDEX created (created);""")
- cursor.execute('UPDATE GBInfo SET created = CURRENT_TIMESTAMP;')
- connection.commit()
- connection.close()
- migration.info('Added column mutalyzer.GBInfo.created')
- migration.info('Added index on mutalyzer.GBInfo.created')
-
-
-if __name__ == '__main__':
- migration.main(check, migrate)
diff --git a/extras/migrations/002-db-map-to-mapping.migration b/extras/migrations/002-db-map-to-mapping.migration
deleted file mode 100755
index 0c0f02af6788b653d834764a8103441bc06ac571..0000000000000000000000000000000000000000
--- a/extras/migrations/002-db-map-to-mapping.migration
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/env python
-"""
-Convert the old 'map' tables to the new 'Mapping' tables.
-
-Usage:
- ./002-db-map-to-mapping.migration [migrate]
-
-This is basically just a renaming of columns and
-- use NULL for missing values
-- add 1 to all chromosomal start positions.
-
-The following tables on hg18 and hg19 are dropped:
-- gbStatus
-- map_cdsBackup
-- refGene
-- refLink
-
-The map tables are renamed to map_backup.
-"""
-
-
-import MySQLdb
-import migration
-
-
-def _exon_starts(starts):
- updated = []
- for start in starts.split(',')[:-1]:
- updated.append(str(int(start) + 1))
- return ','.join(updated)
-
-
-def _exon_stops(stops):
- if stops[-1] == ',':
- return stops[:-1]
-
-
-def _check(db):
- # Todo: Also check if 'map' is gone.
- connection = migration.db_connect(db)
- cursor = connection.cursor()
- cursor.execute('SHOW TABLES LIKE "Mapping";')
- ok = len(cursor.fetchall()) > 0
- connection.close()
- return ok
-
-
-def _migrate(db):
- connection = migration.db_connect(db)
- cursor = connection.cursor()
- cursor.execute("""
- CREATE TABLE Mapping (
- gene varchar(255) DEFAULT NULL,
- transcript varchar(20) NOT NULL DEFAULT '',
- version smallint(6) DEFAULT NULL,
- chromosome varchar(40) DEFAULT NULL,
- orientation char(1) DEFAULT NULL,
- start int(11) unsigned DEFAULT NULL,
- stop int(11) unsigned DEFAULT NULL,
- cds_start int(11) unsigned DEFAULT NULL,
- cds_stop int(11) unsigned DEFAULT NULL,
- exon_starts longblob NOT NULL,
- exon_stops longblob NOT NULL,
- protein varchar(20) DEFAULT NULL,
- source varchar(20) DEFAULT NULL,
- INDEX (transcript)
- );""")
- select_cursor = connection.cursor(MySQLdb.cursors.DictCursor)
- select_cursor.execute("""
- SELECT
- geneName as gene,
- acc as transcript,
- version as version,
- chrom as chromosome,
- strand as orientation,
- txStart + 1 as start,
- txEnd as stop,
- NULLIF(cdsStart + 1, cdsEnd + 1) as cds_start,
- NULLIF(cdsEnd, cdsStart) as cds_stop,
- exonStarts as exon_starts,
- exonEnds as exon_stops,
- NULLIF(protAcc, '') as protein,
- 'UCSC' as source
- FROM
- map;""")
- count = 0
- while True:
- r = select_cursor.fetchone()
- if r == None:
- break
- count += 1
- cursor.execute("""
- INSERT INTO Mapping
- (gene, transcript, version, chromosome, orientation, start, stop,
- cds_start, cds_stop, exon_starts, exon_stops, protein, source)
- VALUES
- (%s, %s, %s, %s, %s, %s, %s,
- %s, %s, %s, %s, %s, %s);""",
- (r['gene'], r['transcript'], r['version'], r['chromosome'],
- r['orientation'], r['start'], r['stop'], r['cds_start'],
- r['cds_stop'], _exon_starts(r['exon_starts']), _exon_stops(r['exon_stops']),
- r['protein'], r['source']))
-
- migration.info('Converted table map to table Mapping on %s (%d entries)' % (db, count))
-
- cursor.execute('DROP TABLE IF EXISTS gbStatus, map_cdsBackup, refGene, refLink')
- cursor.execute('RENAME TABLE map TO map_backup')
-
- migration.info('Dropped tables gbStatus, map_cdsBackup, refGene, refLink on %s' % db)
- migration.info('Renamed table map to map_backup on %s' % db)
-
- select_cursor.close()
- cursor.close()
- connection.commit()
- connection.close()
-
-
-def check():
- """
- Check if migration is needed.
- """
- hg18_ok = _check('hg18')
- hg19_ok = _check('hg19')
- if hg18_ok != hg19_ok:
- migration.fatal('Installation is not in a recognizable state. Fix manually.')
- return not hg18_ok
-
-
-def migrate():
- """
- Perform migration.
- """
- _migrate('hg18')
- _migrate('hg19')
-
-
-if __name__ == '__main__':
- migration.main(check, migrate)
diff --git a/extras/migrations/003-config-remove-ucsc.migration b/extras/migrations/003-config-remove-ucsc.migration
deleted file mode 100755
index 0578d8ce728294ebb33e1abb699593ec9a46819e..0000000000000000000000000000000000000000
--- a/extras/migrations/003-config-remove-ucsc.migration
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-# Remove UCSC database values from the configuration file.
-#
-# Usage:
-# ./003-config-remove-ucsc.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && $(grep -q 'MySQL username for the UCSC database' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- sed -i '/MySQL username for the UCSC database/d' /etc/mutalyzer/config
- sed -i '/Host name for the UCSC database/d' /etc/mutalyzer/config
- sed -i '/Retrieve all entries modified within a certain number of days/d' /etc/mutalyzer/config
- sed -i '/RemoteMySQLuser =/d' /etc/mutalyzer/config
- sed -i '/^RemoteMySQLhost =/d' /etc/mutalyzer/config
- sed -i '/^UpdateInterval =/d' /etc/mutalyzer/config
- echo -e "${COLOR_INFO}Removed all UCSC database configuration values from /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/004-cron-ucsc-to-ncbi.migration b/extras/migrations/004-cron-ucsc-to-ncbi.migration
deleted file mode 100755
index 192bbd297468d3d261c2dc79cf9ac144d9720c85..0000000000000000000000000000000000000000
--- a/extras/migrations/004-cron-ucsc-to-ncbi.migration
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Remove UCSC update from cron and install NCBI update.
-#
-# Usage:
-# ./004-cron-ucsc-to-ncbi.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/cron.d/mutalyzer-ucsc-update ] && $(grep -v -q '^#' /etc/cron.d/mutalyzer-ucsc-update); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- sed -i 's/^/#/' /etc/cron.d/mutalyzer-ucsc-update
- echo -e "${COLOR_INFO}Commented all lines in /etc/cron.d/mutalyzer-ucsc-update${COLOR_END}"
- if [ ! -e /etc/cron.d/mutalyzer-mapping-update ]; then
- BIN_MAPPING_UPDATE=$(which mutalyzer-mapping-update)
- cp extras/cron.d/mutalyzer-mapping-update /etc/cron.d/mutalyzer-mapping-update
- sed -i -e "s@<MUTALYZER_BIN_MAPPING_UPDATE>@${BIN_MAPPING_UPDATE}@g" /etc/cron.d/mutalyzer-mapping-update
- echo -e "${COLOR_INFO}Installed /etc/cron.d/mutalyzer-mapping-update${COLOR_END}"
- fi
- echo 'Performed migration.'
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/005-config-batch-restriction-sites.migration b/extras/migrations/005-config-batch-restriction-sites.migration
deleted file mode 100755
index 36a040fc970113c0a3142aae50a609cfcc6543be..0000000000000000000000000000000000000000
--- a/extras/migrations/005-config-batch-restriction-sites.migration
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# Add batch checker restriction sites headers to the configuration file.
-#
-# Usage:
-# ./005-config-batch-restriction-sites.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && $(grep -q '^nameCheckOutHeader = "Input", "Errors | Messages", "AccNo", "Genesymbol", "Variant", "Reference Sequence Start Descr.", "Coding DNA Descr.", "Protein Descr.", "GeneSymbol Coding DNA Descr.", "GeneSymbol Protein Descr.", "Genomic Reference", "Coding Reference", "Protein Reference", "Affected Transcripts", "Affected Proteins"$' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- sed -i 's/nameCheckOutHeader = "Input", "Errors | Messages", "AccNo", "Genesymbol", "Variant", "Reference Sequence Start Descr.", "Coding DNA Descr.", "Protein Descr.", "GeneSymbol Coding DNA Descr.", "GeneSymbol Protein Descr.", "Genomic Reference", "Coding Reference", "Protein Reference", "Affected Transcripts", "Affected Proteins"/nameCheckOutHeader = "Input", "Errors | Messages", "AccNo", "Genesymbol", "Variant", "Reference Sequence Start Descr.", "Coding DNA Descr.", "Protein Descr.", "GeneSymbol Coding DNA Descr.", "GeneSymbol Protein Descr.", "Genomic Reference", "Coding Reference", "Protein Reference", "Affected Transcripts", "Affected Proteins", "Restriction Sites Created", "Restriction Sites Deleted"/' /etc/mutalyzer/config
- echo -e "${COLOR_INFO}Added batch checker restriction sites headers to /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/006-config-add-autoreconnect.migration b/extras/migrations/006-config-add-autoreconnect.migration
deleted file mode 100755
index 6c9a66f75a0a885777afe0e43587fe7587f54e68..0000000000000000000000000000000000000000
--- a/extras/migrations/006-config-add-autoreconnect.migration
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# Add 'auto reconnect to MySQL server' setting to the configuration file.
-#
-# Usage:
-# ./006-config-add-autoreconnect.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && ! $(grep -q 'autoReconnect' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- # Insert after LocalMySQLhost line
- if $(grep -q '^LocalMySQLhost' /etc/mutalyzer/config && sed -i '/^LocalMySQLhost/ a\
-\
-# Automatically reconnect to MySQL server using the MySQLdb reconnect option.\
-# Note that this may not always be available and if not will result in an\
-# error if used. Mutalyzer also implements its own reconnecting mechanism now\
-# which is always on. See Trac issue #91.\
-autoReconnect = yes' /etc/mutalyzer/config); then
- echo -e "${COLOR_INFO}Added autoReconnect = yes to /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- else
- echo -e "${COLOR_ERROR}Could not perform migration.${COLOR_END}"
- fi
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/007-apache-json-webservice.migration b/extras/migrations/007-apache-json-webservice.migration
deleted file mode 100755
index e4bf006ba58ed711db7b05df346a14e1b3698428..0000000000000000000000000000000000000000
--- a/extras/migrations/007-apache-json-webservice.migration
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Add HTTP/RPC+JSON web service entry to Apache configuration.
-#
-# Usage:
-# ./007-apache-json-webservice.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-BIN_SOAP_SERVICE=$(which mutalyzer-soap-service.wsgi)
-BIN_JSON_SERVICE=$(which mutalyzer-json-service.wsgi)
-
-BIN_OLD_SOAP_SERVICE=$(echo $BIN_SOAP_SERVICE | sed 's/mutalyzer-soap-service/mutalyzer-webservice/')
-
-CONF=/etc/apache2/conf.d/mutalyzer.conf
-
-if [ -e $CONF ] && ! $(grep -q "${BIN_JSON_SERVICE}" $CONF) && $(grep -q "${BIN_OLD_SOAP_SERVICE}" $CONF); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- if $(grep -q '^# Website' $CONF); then
- sed -i 's!^# Webservice!# SOAP/1.1 webservice!' $CONF
- sed -i "s!$BIN_OLD_SOAP_SERVICE!$BIN_SOAP_SERVICE!g" $CONF
- echo -e "${COLOR_INFO}Changed ${BIN_OLD_SOAP_SERVICE} to ${BIN_SOAP_SERVICE} in ${CONF}${COLOR_END}"
- # Insert before '# Website' line
- sed -i "/^# Website/ i\\
-# HTTP/RPC+JSON webservice\\
-WSGIScriptAlias /mutalyzer/json $BIN_JSON_SERVICE\\
-<Directory /mutalyzer/json>\\
- Order deny,allow\\
- Allow from all\\
- Options -Indexes\\
-</Directory>\\
-" $CONF
- echo -e "${COLOR_INFO}Added HTTP/RPC+JSON webservice at /mutalyzer/json to ${CONF}${COLOR_END}"
- echo 'Performed migration.'
- else
- echo -e "${COLOR_ERROR}Could not perform migration.${COLOR_END}"
- fi
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/008-config-add-piwik.migration b/extras/migrations/008-config-add-piwik.migration
deleted file mode 100755
index 2ee67afad3e0ade541ce0ef1f910d3bf46769261..0000000000000000000000000000000000000000
--- a/extras/migrations/008-config-add-piwik.migration
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Add Piwik analytics settings (disabled by default) to the configuration file.
-#
-# Usage:
-# ./008-config-add-piwik.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && ! $(grep -q 'piwik' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- # Insert Piwik settings at the bottom
- cat <<EOF >> /etc/mutalyzer/config
-
-
-#
-# These settings are for Piwik analytics.
-#
-
-# Enable Piwik analytics.
-piwik = no
-
-# Piwik server base URL (include protocol, no trailing slash).
-piwikBase = https://piwik.example.com
-
-# Piwik site ID.
-piwikSite = 1
-EOF
- echo -e "${COLOR_INFO}Added Piwik settings to /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/009-db-mapping-add-selector.migration b/extras/migrations/009-db-mapping-add-selector.migration
deleted file mode 100755
index 21b918f65c740fd9a5c96080f5938f0aa2d9b144..0000000000000000000000000000000000000000
--- a/extras/migrations/009-db-mapping-add-selector.migration
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-"""
-Add columns 'selector' and 'selector_version' to the 'Mapping' tables.
-
-Usage:
- ./009-db-mapping-add-selector.migration [migrate]
-"""
-
-
-import migration
-
-
-def check():
- """
- Check if migration is needed.
- """
- connection = migration.db_connect('hg18')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM Mapping WHERE field = "selector";')
- has_column_hg18 = len(cursor.fetchall()) > 0
- connection.close()
-
- connection = migration.db_connect('hg19')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM Mapping WHERE field = "selector";')
- has_column_hg19 = len(cursor.fetchall()) > 0
- connection.close()
-
- if has_column_hg19 != has_column_hg19:
- migration.fatal('Installation is not in a recognizable state. Fix manually.')
- return not has_column_hg19
-
-
-def migrate():
- """
- Perform migration.
- """
- connection = migration.db_connect('hg18')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE Mapping
- ADD COLUMN selector varchar(255) DEFAULT NULL AFTER version,
- ADD COLUMN selector_version smallint(6) DEFAULT NULL AFTER selector;""")
- connection.commit()
- connection.close()
- migration.info('Added column hg18.Mapping.selector')
- migration.info('Added column hg18.Mapping.selector_version')
-
- connection = migration.db_connect('hg19')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE Mapping
- ADD COLUMN selector varchar(255) DEFAULT NULL AFTER version,
- ADD COLUMN selector_version smallint(6) DEFAULT NULL AFTER selector;""")
- connection.commit()
- connection.close()
- migration.info('Added column hg19.Mapping.selector')
- migration.info('Added column hg19.Mapping.selector_version')
-
-
-if __name__ == '__main__':
- migration.main(check, migrate)
diff --git a/extras/migrations/010-db-chrname-add-organelle.migration b/extras/migrations/010-db-chrname-add-organelle.migration
deleted file mode 100755
index c7e541af02cb8bc03b021697fa4696b474a9abf6..0000000000000000000000000000000000000000
--- a/extras/migrations/010-db-chrname-add-organelle.migration
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-"""
-Add a column 'organelle_type' to the 'ChrName' tables.
-
-Usage:
- ./010-db-chrname-add-organelle.migration [migrate]
-"""
-
-
-import migration
-
-
-def check():
- """
- Check if migration is needed.
- """
- connection = migration.db_connect('hg18')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM ChrName WHERE field = "organelle_type";')
- has_column_hg18 = len(cursor.fetchall()) > 0
- connection.close()
-
- connection = migration.db_connect('hg19')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM ChrName WHERE field = "organelle_type";')
- has_column_hg19 = len(cursor.fetchall()) > 0
- connection.close()
-
- if has_column_hg19 != has_column_hg19:
- migration.fatal('Installation is not in a recognizable state. Fix manually.')
- return not has_column_hg19
-
-
-def migrate():
- """
- Perform migration.
- """
- connection = migration.db_connect('hg18')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE ChrName
- ADD COLUMN organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome' AFTER name;""")
- connection.commit()
- cursor = connection.cursor()
- cursor.execute("""
- UPDATE ChrName SET organelle_type = 'mitochondrion' WHERE name = 'chrM';""")
- connection.commit()
- connection.close()
- migration.info('Added column hg18.ChrName.organelle_type')
-
- connection = migration.db_connect('hg19')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE ChrName
- ADD COLUMN organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome' AFTER name;""")
- connection.commit()
- cursor = connection.cursor()
- cursor.execute("""
- UPDATE ChrName SET organelle_type = 'mitochondrion' WHERE name = 'chrM';""")
- connection.commit()
- connection.close()
- migration.info('Added column hg19.ChrName.organelle_type')
-
-
-if __name__ == '__main__':
- migration.main(check, migrate)
diff --git a/extras/migrations/011-config-add-defaultdb.migration b/extras/migrations/011-config-add-defaultdb.migration
deleted file mode 100755
index 9f845c3c06773d3ba6c95e278b36276fe6fdcd53..0000000000000000000000000000000000000000
--- a/extras/migrations/011-config-add-defaultdb.migration
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Add 'default mapping database' setting to the configuration file.
-#
-# Usage:
-# ./011-config-add-defaultdb.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && ! $(grep -q 'defaultDb' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- # Insert after dbNames line
- if $(grep -q '^dbNames' /etc/mutalyzer/config && sed -i '/^dbNames/ a\
-\
-# Default mapping database.\
-defaultDb = "hg19"' /etc/mutalyzer/config); then
- echo -e "${COLOR_INFO}Added defaultDb = \"hg19\" to /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- else
- echo -e "${COLOR_ERROR}Could not perform migration.${COLOR_END}"
- fi
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/012-db-add-mm10.migration b/extras/migrations/012-db-add-mm10.migration
deleted file mode 100755
index 180dd2621c5d63d3c02d4eba0cb33d60a850dc6c..0000000000000000000000000000000000000000
--- a/extras/migrations/012-db-add-mm10.migration
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/bin/bash
-
-# Add mm10 database with mouse mappings.
-#
-# Usage:
-# ./012-db-add-mm10.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if echo 'show databases;' | mysql -u mutalyzer | grep -q mm10; then
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-else
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
-
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
-
- echo -e "${COLOR_INFO}Creating mm10 database${COLOR_END}"
-
- echo "You will now be asked for the MySQL root password"
- # Create databases
- cat << EOF | mysql -u root -p
- CREATE DATABASE mm10;
- GRANT ALL PRIVILEGES ON mm10.* TO mutalyzer@localhost;
- FLUSH PRIVILEGES;
-EOF
-
- if [ $? -ne 0 ]; then
- echo -e "${COLOR_ERROR}Error creating mm10 database, aborting migration.${COLOR_END}"
- exit 1
- fi
-
- echo -e "${COLOR_INFO}Creating tables in mm10 database${COLOR_END}"
-
- # Create ChrName and Mapping table (mm10)
- cat << EOF | mysql -u mutalyzer -D mm10
- CREATE TABLE ChrName (
- AccNo char(20) NOT NULL,
- name char(20) NOT NULL,
- organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome',
- PRIMARY KEY (AccNo)
- ) ENGINE = MYISAM;
- CREATE TABLE Mapping (
- gene varchar(255) DEFAULT NULL,
- transcript varchar(20) NOT NULL DEFAULT '',
- version smallint(6) DEFAULT NULL,
- selector varchar(255) DEFAULT NULL,
- selector_version smallint(6) DEFAULT NULL,
- chromosome varchar(40) DEFAULT NULL,
- orientation char(1) DEFAULT NULL,
- start int(11) unsigned DEFAULT NULL,
- stop int(11) unsigned DEFAULT NULL,
- cds_start int(11) unsigned DEFAULT NULL,
- cds_stop int(11) unsigned DEFAULT NULL,
- exon_starts longblob NOT NULL,
- exon_stops longblob NOT NULL,
- protein varchar(20) DEFAULT NULL,
- source varchar(20) DEFAULT NULL,
- INDEX (transcript)
- ) ENGINE = MYISAM;
- INSERT INTO ChrName (AccNo, name, organelle_type) VALUES
- ('NC_000067.65', 'chr1', 'chromosome'),
- ('NC_000068.70', 'chr2', 'chromosome'),
- ('NC_000069.60', 'chr3', 'chromosome'),
- ('NC_000070.66', 'chr4', 'chromosome'),
- ('NC_000071.65', 'chr5', 'chromosome'),
- ('NC_000072.60', 'chr6', 'chromosome'),
- ('NC_000073.61', 'chr7', 'chromosome'),
- ('NC_000074.60', 'chr8', 'chromosome'),
- ('NC_000075.60', 'chr9', 'chromosome'),
- ('NC_000076.60', 'chr10', 'chromosome'),
- ('NC_000077.60', 'chr11', 'chromosome'),
- ('NC_000078.60', 'chr12', 'chromosome'),
- ('NC_000079.60', 'chr13', 'chromosome'),
- ('NC_000080.60', 'chr14', 'chromosome'),
- ('NC_000081.60', 'chr15', 'chromosome'),
- ('NC_000082.60', 'chr16', 'chromosome'),
- ('NC_000083.60', 'chr17', 'chromosome'),
- ('NC_000084.60', 'chr18', 'chromosome'),
- ('NC_000085.60', 'chr19', 'chromosome'),
- ('NC_000086.71', 'chrX', 'chromosome'),
- ('NC_000087.74', 'chrY', 'chromosome'),
- ('NC_005089.1', 'chrM', 'mitochondrion');
-EOF
-
- if [ $? -ne 0 ]; then
- echo -e "${COLOR_ERROR}Error creating tables in mm10 database, aborting migration.${COLOR_END}"
- exit 1
- fi
-
- echo -e "${COLOR_INFO}Populating Mapping table with NCBI data (mm10)${COLOR_END}"
-
- # Populate Mapping table with UCSC data (mm10)
- MAPPING=$(mktemp)
- wget "ftp://ftp.ncbi.nih.gov/genomes/M_musculus/ARCHIVE/BUILD.38.1/mapview/seq_gene.md.gz" -O - | zcat > $MAPPING
- echo -e "${COLOR_INFO}Importing NCBI mapping data, this may take a few minutes (mm10)${COLOR_END}"
- BIN_MAPPING_UPDATE=$(which mutalyzer-mapping-update)
- $($BIN_MAPPING_UPDATE mm10 $MAPPING 'GRCm38-C57BL/6J')
- rm $MAPPING
-
- echo -e "${COLOR_INFO}Separate import of NC_005089.1 (chrM) in mm10 database${COLOR_END}"
- BIN_MAPPING_IMPORT=$(which mutalyzer-mapping-import)
- $($BIN_MAPPING_IMPORT reference mm10 NC_005089.1)
-
- if [ -e /etc/mutalyzer/config ] && $(grep '^dbNames' /etc/mutalyzer/config | grep -q -v mm10); then
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- sed -i 's/^\(dbNames.*\)$/\1, "mm10"/' /etc/mutalyzer/config
- echo -e "${COLOR_INFO}Added mm10 database to /etc/mutalyzer/config${COLOR_END}"
- fi
-
- echo 'Performed migration.'
- fi
-fi
diff --git a/extras/migrations/013-db-add-counter.migration b/extras/migrations/013-db-add-counter.migration
deleted file mode 100755
index eb312b808f97cd2d4282f752e688f62b21aea314..0000000000000000000000000000000000000000
--- a/extras/migrations/013-db-add-counter.migration
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-# Add Counter table to mutalyzer database.
-#
-# Usage:
-# ./013-db-add-counter.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if echo 'show tables;' | mysql -u mutalyzer -D mutalyzer | grep -q Counter; then
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-else
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
-
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
-
- echo -e "${COLOR_INFO}Creating mutalyzer.Counter table${COLOR_END}"
-
- # Create Counter table and initial rows
- cat << EOF | mysql -u mutalyzer -D mutalyzer
-CREATE TABLE Counter (
- service varchar(100) NOT NULL,
- interface varchar(100) NOT NULL,
- count bigint NOT NULL DEFAULT 0,
- start TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (service, interface)
-) ENGINE = MYISAM;
-INSERT INTO Counter (service, interface) VALUES
-('namecheck', 'website'),
-('syntaxcheck', 'website'),
-('positionconvert', 'website'),
-('snpconvert', 'website'),
-('batchjob', 'website'),
-('namecheck', 'webservice'),
-('syntaxcheck', 'webservice'),
-('positionconvert', 'webservice'),
-('snpconvert', 'webservice'),
-('batchjob', 'webservice'),
-('namecheck', 'batch'),
-('syntaxcheck', 'batch'),
-('positionconvert', 'batch'),
-('snpconvert', 'batch');
-EOF
-
- echo 'Performed migration.'
- fi
-fi
diff --git a/extras/migrations/014-db-link-add-created.migration b/extras/migrations/014-db-link-add-created.migration
deleted file mode 100755
index 486deff39eb59f29e2b066bc55e6181af636bed4..0000000000000000000000000000000000000000
--- a/extras/migrations/014-db-link-add-created.migration
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-"""
-Add a column 'created' and alter column 'protAcc' to allow NULL values on the
-'Link' table.
-
-Usage:
- ./014-db-link-add-created.migration [migrate]
-"""
-
-
-import migration
-
-
-def check():
- """
- Check if migration is needed.
- """
- connection = migration.db_connect('mutalyzer')
- cursor = connection.cursor()
- cursor.execute('SHOW COLUMNS FROM Link WHERE field = "created";')
- has_column = len(cursor.fetchall()) > 0
- connection.close()
- return not has_column
-
-
-def migrate():
- """
- Perform migration.
- """
- connection = migration.db_connect('mutalyzer')
- cursor = connection.cursor()
- cursor.execute("""
- ALTER TABLE Link
- MODIFY COLUMN protAcc CHAR(20) DEFAULT NULL;""")
- cursor.execute("""
- ALTER TABLE Link
- ADD COLUMN created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;""")
- cursor.execute('UPDATE Link SET created = CURRENT_TIMESTAMP;')
- connection.commit()
- connection.close()
- migration.info('Added column mutalyzer.Link.created')
- migration.info('Altered column mutalyzer.Link.protAcc to allow NULL values')
-
-
-if __name__ == '__main__':
- migration.main(check, migrate)
diff --git a/extras/migrations/015-config-add-proteinlinklifetime.migration b/extras/migrations/015-config-add-proteinlinklifetime.migration
deleted file mode 100755
index ee86e4aeac861889d5b8603f8b2f3583b7888a1c..0000000000000000000000000000000000000000
--- a/extras/migrations/015-config-add-proteinlinklifetime.migration
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Add 'protein link lifetime' and 'protein link (none) lifetime' settings to
-# the configuration file.
-#
-# Usage:
-# ./015-config-add-proteinlinklifetime.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-if [ -e /etc/mutalyzer/config ] && ! $(grep -q 'proteinLinkLifetime' /etc/mutalyzer/config); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- echo -e "${COLOR_INFO}Copying /etc/mutalyzer/config to /etc/mutalyzer/config.backup${COLOR_END}"
- cp /etc/mutalyzer/config /etc/mutalyzer/config.backup
- # Insert after autoReconnect line
- if $(grep -q '^autoReconnect' /etc/mutalyzer/config && sed -i '/^autoReconnect/ a\
-\
-# Number of days a cached transcript->protein link from the NCBI is valid.\
-proteinLinkLifetime = 30\
-\
-# Number of days a cached nonexisting transcript->protein link from the NCBI\
-# is valid.\
-proteinLinkNoneLifetime = 5' /etc/mutalyzer/config); then
- echo -e "${COLOR_INFO}Added proteinLinkLifetime = 30 to /etc/mutalyzer/config${COLOR_END}"
- echo -e "${COLOR_INFO}Added proteinLinkNoneLifetime = 5 to /etc/mutalyzer/config${COLOR_END}"
- echo 'Performed migration.'
- else
- echo -e "${COLOR_ERROR}Could not perform migration.${COLOR_END}"
- fi
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/016-apache-static.migration b/extras/migrations/016-apache-static.migration
deleted file mode 100755
index 7dd68114ced5113340b998b2f2eb3ad29e352f13..0000000000000000000000000000000000000000
--- a/extras/migrations/016-apache-static.migration
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# Update Apache configuration from /base to /static
-#
-# Usage:
-# ./016-apache-static.migration [migrate]
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-CONF=/etc/apache2/conf.d/mutalyzer.conf
-
-if [ -e $CONF ] && ! $(grep -q "/mutalyzer/static" $CONF) && $(grep -q "/mutalyzer/base" $CONF); then
- echo -e "${COLOR_WARNING}This migration is needed.${COLOR_END}"
- if [ "$1" = 'migrate' ]; then
- echo 'Performing migration.'
- sed -i "s!/mutalyzer/base!/mutalyzer/static!g" $CONF
- echo -e "${COLOR_INFO}Changed /mutalyzer/base to /mutalyzer/static in ${CONF}${COLOR_END}"
- echo 'Performed migration.'
- fi
-else
- echo -e "${COLOR_INFO}This migration is not needed.${COLOR_END}"
-fi
diff --git a/extras/migrations/README b/extras/migrations/README
deleted file mode 100644
index e514423f3eda8b60a79e6aa430fa8984bfd9085d..0000000000000000000000000000000000000000
--- a/extras/migrations/README
+++ /dev/null
@@ -1,18 +0,0 @@
-Automated migration scripts
-===========================
-
-This directory contains scripts to automate the migration of a Mutalyzer
-installation to the latest version. Things that might need a migration
-include:
-
-- database schema
-- database data
-- configuration files
-- cache
-- ?
-
-All migration scripts accept as parameters a flag 'migrate'. Running a
-script without parameters just checks if the migration is needed. Running
-a script with 'migrate' does the actual migration (only if needed).
-
-Performing multiple migrations should be done in the order of their names.
diff --git a/extras/migrations/migration.py b/extras/migrations/migration.py
deleted file mode 100644
index c3e6b3dd935e7bad983f9371e372f9bdf7a2a838..0000000000000000000000000000000000000000
--- a/extras/migrations/migration.py
+++ /dev/null
@@ -1,62 +0,0 @@
-"""
-Some utility functions for our simple migrations.
-
-@todo: Perhaps this should be moved to the mutalyzer package.
-"""
-
-
-import sys
-import MySQLdb
-
-
-COLOR_INFO = '\033[32m'
-COLOR_WARNING = '\033[33m'
-COLOR_ERROR = '\033[31m'
-COLOR_END = '\033[0m'
-
-
-def print_color(message, color=None):
- if color is None:
- print message
- else:
- print color + message + COLOR_END
-
-
-def info(message):
- print_color(message, COLOR_INFO)
-
-
-def warning(message):
- print_color(message, COLOR_WARNING)
-
-
-def error(message):
- print_color(message, COLOR_ERROR)
-
-
-def fatal(message):
- error(message)
- sys.exit(1)
-
-
-def db_connect(database):
- try:
- connection = MySQLdb.connect(host='localhost',
- user='mutalyzer',
- passwd='',
- db=database)
- except MySQLdb.Error as e:
- fatal('Error %d: %s' % (e.args[0], e.args[1]))
- return connection
-
-
-def main(check, migrate):
- needed = check()
- if needed:
- warning('This migration is needed.')
- if len(sys.argv) > 1 and sys.argv[1] == 'migrate':
- print 'Performing migration.'
- migrate()
- print 'Performed migration.'
- else:
- info('This migration is not needed.')
diff --git a/extras/post-install.sh b/extras/post-install.sh
deleted file mode 100644
index 2b5e74680ce71de5a5bc9812c15c431f08ce5693..0000000000000000000000000000000000000000
--- a/extras/post-install.sh
+++ /dev/null
@@ -1,407 +0,0 @@
-#!/bin/bash
-
-# Post-install script for Mutalyzer. Run after the setuptools installation
-# (python setup.py install).
-#
-# Notice: The definitions in this file are quite specific to the standard
-# Mutalyzer environment. This consists of a Debian stable (Squeeze) system
-# with Apache and Mutalyzer using its mod_wsgi module. Debian conventions are
-# used throughout. See the README file for more information.
-#
-# Usage (from the source root directory):
-# sudo bash extras/post-install.sh
-#
-# Todo:
-# - Copy doc to /usr/share/doc
-# - Check if MySQL user/database exist before creating
-# - General cleanup
-
-set -e
-set -u
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-# The 'cd /' is a hack to prevent the mutalyzer package under the current
-# directory to be used.
-STATIC_DIR=$(cd / && python -c 'import pkg_resources; print pkg_resources.resource_filename("mutalyzer", "templates/static")')
-BIN_BATCHD=$(which mutalyzer-batchd)
-BIN_CACHE_SYNC=$(which mutalyzer-cache-sync)
-BIN_MAPPING_UPDATE=$(which mutalyzer-mapping-update)
-BIN_MAPPING_IMPORT=$(which mutalyzer-mapping-import)
-BIN_WEBSITE=$(which mutalyzer-website.wsgi)
-BIN_SOAP_SERVICE=$(which mutalyzer-soap-service.wsgi)
-BIN_JSON_SERVICE=$(which mutalyzer-json-service.wsgi)
-
-if [ ! -e /etc/mutalyzer/config ]; then
- echo -e "${COLOR_INFO}Creating /etc/mutalyzer/config${COLOR_END}"
- mkdir -p /etc/mutalyzer
- cp extras/config.example /etc/mutalyzer/config
- chmod -R u=rwX,go=rX /etc/mutalyzer
-else
- echo -e "${COLOR_WARNING}Not touching /etc/mutalyzer/config (it exists)${COLOR_END}"
-fi
-
-for USERNAME in $(cut -f 1 -d : /etc/passwd); do
- if [ -d "/home/${USERNAME}" ]; then
- echo -e "${COLOR_INFO}Creating /home/${USERNAME}/.config/mutalyzer/config${COLOR_END}"
- echo -e "${COLOR_INFO}Creating /home/${USERNAME}/.cache/mutalyzer${COLOR_END}"
- # Instead of su, use chown later.
- su $USERNAME -c "mkdir -p /home/$USERNAME/.config/mutalyzer"
- su $USERNAME -c "mkdir -p /home/$USERNAME/.cache/mutalyzer"
- su $USERNAME -c "touch /home/$USERNAME/.config/mutalyzer/config"
- su $USERNAME -c "touch /tmp/mutalyzer-$USERNAME.log"
- cat extras/config.user.example > /home/$USERNAME/.config/mutalyzer/config
- sed -i -e "s@<USERNAME>@${USERNAME}@g" /home/$USERNAME/.config/mutalyzer/config
- fi
-done
-
-echo -e "${COLOR_INFO}Touching /var/log/mutalyzer.log${COLOR_END}"
-touch /var/log/mutalyzer.log
-chown www-data:www-data /var/log/mutalyzer.log
-chmod u=rw,go=r /var/log/mutalyzer.log
-
-echo -e "${COLOR_INFO}Touching /var/cache/mutalyzer${COLOR_END}"
-mkdir -p /var/cache/mutalyzer
-chown -R www-data:www-data /var/cache/mutalyzer
-chmod -R u=rwX,go=rX /var/cache/mutalyzer
-
-echo -e "${COLOR_INFO}Creating /etc/init.d/mutalyzer-batchd${COLOR_INFO}"
-cp extras/init.d/mutalyzer-batchd /etc/init.d/mutalyzer-batchd
-sed -i -e "s@<MUTALYZER_BIN_BATCHD>@${BIN_BATCHD}@g" /etc/init.d/mutalyzer-batchd
-chmod u=rwx,go=rx /etc/init.d/mutalyzer-batchd
-
-echo -e "${COLOR_INFO}Installing init scripts${COLOR_END}"
-update-rc.d -f mutalyzer-batchd remove
-update-rc.d mutalyzer-batchd defaults 98 02
-
-echo -e "${COLOR_INFO}Installing crontab${COLOR_END}"
-cp extras/cron.d/mutalyzer-cache-sync /etc/cron.d/mutalyzer-cache-sync
-sed -i -e "s@<MUTALYZER_BIN_CACHE_SYNC>@${BIN_CACHE_SYNC}@g" /etc/cron.d/mutalyzer-cache-sync
-cp extras/cron.d/mutalyzer-mapping-update /etc/cron.d/mutalyzer-mapping-update
-sed -i -e "s@<MUTALYZER_BIN_MAPPING_UPDATE>@${BIN_MAPPING_UPDATE}@g" /etc/cron.d/mutalyzer-mapping-update
-
-echo -e "${COLOR_INFO}Creating /etc/apache2/conf.d/mutalyzer.conf${COLOR_END}"
-cp extras/apache/mutalyzer.conf /etc/apache2/conf.d/mutalyzer.conf
-sed -i -e "s@<MUTALYZER_BIN_WEBSITE>@${BIN_WEBSITE}@g" -e "s@<MUTALYZER_BIN_SOAP_SERVICE>@${BIN_SOAP_SERVICE}@g" -e "s@<MUTALYZER_BIN_JSON_SERVICE>@${BIN_JSON_SERVICE}@g" -e "s@<MUTALYZER_BIN_BATCHD>@${BIN_BATCHD}@g" /etc/apache2/conf.d/mutalyzer.conf
-chmod u=rw,go=r /etc/apache2/conf.d/mutalyzer.conf
-
-echo -e "${COLOR_INFO}Creating databases${COLOR_END}"
-
-echo "You will now be asked for the MySQL root password"
-
-# Create databases
-cat << EOF | mysql -u root -p
- CREATE USER mutalyzer;
- CREATE DATABASE mutalyzer;
- CREATE DATABASE hg18;
- CREATE DATABASE hg19;
- CREATE DATABASE mm10;
- GRANT ALL PRIVILEGES ON mutalyzer.* TO mutalyzer@localhost;
- GRANT ALL PRIVILEGES ON hg18.* TO mutalyzer@localhost;
- GRANT ALL PRIVILEGES ON hg19.* TO mutalyzer@localhost;
- GRANT ALL PRIVILEGES ON mm10.* TO mutalyzer@localhost;
- FLUSH PRIVILEGES;
-EOF
-
-echo -e "${COLOR_INFO}Creating tables in mutalyzer database${COLOR_END}"
-
-# Create mutalyzer tables
-cat << EOF | mysql -u mutalyzer -D mutalyzer
-CREATE TABLE BatchJob (
- JobID char(20) NOT NULL,
- Filter char(20) NOT NULL,
- EMail char(255) NOT NULL,
- FromHost char(255) NOT NULL,
- JobType char(20) DEFAULT NULL,
- Arg1 char(20) DEFAULT NULL,
- PRIMARY KEY (JobID)
-) ENGINE = MYISAM;
-CREATE TABLE BatchQueue (
- QueueID int(5) NOT NULL AUTO_INCREMENT,
- JobID char(20) NOT NULL,
- Input char(255) NOT NULL,
- Flags char(20) DEFAULT NULL,
- PRIMARY KEY (QueueID),
- KEY JobQueue (JobID,QueueID)
-) ENGINE = MYISAM;
-CREATE TABLE GBInfo (
- AccNo char(20) NOT NULL DEFAULT '',
- GI char(13) DEFAULT NULL,
- hash char(32) NOT NULL DEFAULT '',
- ChrAccVer char(20) DEFAULT NULL,
- ChrStart int(12) DEFAULT NULL,
- ChrStop int(12) DEFAULT NULL,
- orientation int(2) DEFAULT NULL,
- url char(255) DEFAULT NULL,
- created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (AccNo),
- UNIQUE KEY hash (hash),
- UNIQUE KEY alias (GI),
- INDEX (created)
-) ENGINE = MYISAM;
-CREATE TABLE Link (
- mrnaAcc char(20) NOT NULL,
- protAcc char(20) DEFAULT NULL,
- created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (mrnaAcc),
- UNIQUE KEY protAcc (protAcc)
-) ENGINE = MYISAM;
-CREATE TABLE Counter (
- service varchar(100) NOT NULL,
- interface varchar(100) NOT NULL,
- count bigint NOT NULL DEFAULT 0,
- start TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (service, interface)
-) ENGINE = MYISAM;
-INSERT INTO Counter (service, interface) VALUES
-('namecheck', 'website'),
-('syntaxcheck', 'website'),
-('positionconvert', 'website'),
-('snpconvert', 'website'),
-('batchjob', 'website'),
-('namecheck', 'webservice'),
-('syntaxcheck', 'webservice'),
-('positionconvert', 'webservice'),
-('snpconvert', 'webservice'),
-('batchjob', 'webservice'),
-('namecheck', 'batch'),
-('syntaxcheck', 'batch'),
-('positionconvert', 'batch'),
-('snpconvert', 'batch');
-EOF
-
-echo -e "${COLOR_INFO}Creating tables in hg18 database${COLOR_END}"
-
-# Create ChrName and Mapping table (hg18)
-cat << EOF | mysql -u mutalyzer -D hg18
-CREATE TABLE ChrName (
- AccNo char(20) NOT NULL,
- name char(20) NOT NULL,
- organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome',
- PRIMARY KEY (AccNo)
-) ENGINE = MYISAM;
-CREATE TABLE Mapping (
- gene varchar(255) DEFAULT NULL,
- transcript varchar(20) NOT NULL DEFAULT '',
- version smallint(6) DEFAULT NULL,
- selector varchar(255) DEFAULT NULL,
- selector_version smallint(6) DEFAULT NULL,
- chromosome varchar(40) DEFAULT NULL,
- orientation char(1) DEFAULT NULL,
- start int(11) unsigned DEFAULT NULL,
- stop int(11) unsigned DEFAULT NULL,
- cds_start int(11) unsigned DEFAULT NULL,
- cds_stop int(11) unsigned DEFAULT NULL,
- exon_starts longblob NOT NULL,
- exon_stops longblob NOT NULL,
- protein varchar(20) DEFAULT NULL,
- source varchar(20) DEFAULT NULL,
- INDEX (transcript)
-) ENGINE = MYISAM;
-INSERT INTO ChrName (AccNo, name, organelle_type) VALUES
-('NC_000001.9', 'chr1', 'chromosome'),
-('NC_000002.10', 'chr2', 'chromosome'),
-('NC_000003.10', 'chr3', 'chromosome'),
-('NC_000004.10', 'chr4', 'chromosome'),
-('NC_000005.8', 'chr5', 'chromosome'),
-('NC_000006.10', 'chr6', 'chromosome'),
-('NC_000007.12', 'chr7', 'chromosome'),
-('NC_000008.9', 'chr8', 'chromosome'),
-('NC_000009.10', 'chr9', 'chromosome'),
-('NC_000010.9', 'chr10', 'chromosome'),
-('NC_000011.8', 'chr11', 'chromosome'),
-('NC_000012.10', 'chr12', 'chromosome'),
-('NC_000013.9', 'chr13', 'chromosome'),
-('NC_000014.7', 'chr14', 'chromosome'),
-('NC_000015.8', 'chr15', 'chromosome'),
-('NC_000016.8', 'chr16', 'chromosome'),
-('NC_000017.9', 'chr17', 'chromosome'),
-('NC_000018.8', 'chr18', 'chromosome'),
-('NC_000019.8', 'chr19', 'chromosome'),
-('NC_000020.9', 'chr20', 'chromosome'),
-('NC_000021.7', 'chr21', 'chromosome'),
-('NC_000022.9', 'chr22', 'chromosome'),
-('NC_000023.9', 'chrX', 'chromosome'),
-('NC_000024.8', 'chrY', 'chromosome'),
-('NC_001807.4', 'chrM', 'mitochondrion'),
-('NT_113891.1', 'chr6_cox_hap1', 'chromosome'),
-('NT_113959.1', 'chr22_h2_hap1', 'chromosome');
-EOF
-
-echo -e "${COLOR_INFO}Populating Mapping table with NCBI data (hg18)${COLOR_END}"
-
-# Populate Mapping table with NCBI data (hg18)
-MAPPING=$(mktemp)
-wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.36.3/mapview/seq_gene.md.gz" -O - | zcat > $MAPPING
-echo -e "${COLOR_INFO}Importing NCBI mapping data, this may take a few minutes (hg18)${COLOR_END}"
-$($BIN_MAPPING_UPDATE hg18 $MAPPING reference)
-rm $MAPPING
-
-echo -e "${COLOR_INFO}Separate import of NC_001807.4 (chrM) in hg18 database${COLOR_END}"
-$($BIN_MAPPING_IMPORT reference hg18 NC_001807.4)
-
-echo -e "${COLOR_INFO}Creating tables in hg19 database${COLOR_END}"
-
-# Create ChrName and Mapping table (hg19)
-cat << EOF | mysql -u mutalyzer -D hg19
-CREATE TABLE ChrName (
- AccNo char(20) NOT NULL,
- name char(20) NOT NULL,
- organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome',
- PRIMARY KEY (AccNo)
-) ENGINE = MYISAM;
-CREATE TABLE Mapping (
- gene varchar(255) DEFAULT NULL,
- transcript varchar(20) NOT NULL DEFAULT '',
- version smallint(6) DEFAULT NULL,
- selector varchar(255) DEFAULT NULL,
- selector_version smallint(6) DEFAULT NULL,
- chromosome varchar(40) DEFAULT NULL,
- orientation char(1) DEFAULT NULL,
- start int(11) unsigned DEFAULT NULL,
- stop int(11) unsigned DEFAULT NULL,
- cds_start int(11) unsigned DEFAULT NULL,
- cds_stop int(11) unsigned DEFAULT NULL,
- exon_starts longblob NOT NULL,
- exon_stops longblob NOT NULL,
- protein varchar(20) DEFAULT NULL,
- source varchar(20) DEFAULT NULL,
- INDEX (transcript)
-) ENGINE = MYISAM;
-INSERT INTO ChrName (AccNo, name, organelle_type) VALUES
-('NC_000001.10', 'chr1', 'chromosome'),
-('NC_000002.11', 'chr2', 'chromosome'),
-('NC_000003.11', 'chr3', 'chromosome'),
-('NC_000004.11', 'chr4', 'chromosome'),
-('NC_000005.9', 'chr5', 'chromosome'),
-('NC_000006.11', 'chr6', 'chromosome'),
-('NC_000007.13', 'chr7', 'chromosome'),
-('NC_000008.10', 'chr8', 'chromosome'),
-('NC_000009.11', 'chr9', 'chromosome'),
-('NC_000010.10', 'chr10', 'chromosome'),
-('NC_000011.9', 'chr11', 'chromosome'),
-('NC_000012.11', 'chr12', 'chromosome'),
-('NC_000013.10', 'chr13', 'chromosome'),
-('NC_000014.8', 'chr14', 'chromosome'),
-('NC_000015.9', 'chr15', 'chromosome'),
-('NC_000016.9', 'chr16', 'chromosome'),
-('NC_000017.10', 'chr17', 'chromosome'),
-('NC_000018.9', 'chr18', 'chromosome'),
-('NC_000019.9', 'chr19', 'chromosome'),
-('NC_000020.10', 'chr20', 'chromosome'),
-('NC_000021.8', 'chr21', 'chromosome'),
-('NC_000022.10', 'chr22', 'chromosome'),
-('NC_000023.10', 'chrX', 'chromosome'),
-('NC_000024.9', 'chrY', 'chromosome'),
-('NC_012920.1', 'chrM', 'mitochondrion'),
-('NT_167244.1', 'chr6_apd_hap1', 'chromosome'),
-('NT_113891.2', 'chr6_cox_hap2', 'chromosome'),
-('NT_167245.1', 'chr6_dbb_hap3', 'chromosome'),
-('NT_167246.1', 'chr6_mann_hap4', 'chromosome'),
-('NT_167247.1', 'chr6_mcf_hap5', 'chromosome'),
-('NT_167248.1', 'chr6_qbl_hap6', 'chromosome'),
-('NT_167249.1', 'chr6_ssto_hap7', 'chromosome'),
-('NT_167250.1', 'chr4_ctg9_hap1', 'chromosome'),
-('NT_167251.1', 'chr17_ctg5_hap1', 'chromosome');
-EOF
-
-echo -e "${COLOR_INFO}Populating Mapping table with NCBI data (hg19)${COLOR_END}"
-
-# Populate Mapping table with UCSC data (hg19)
-MAPPING=$(mktemp)
-#wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/mapview/seq_gene.md.gz" -O - | zcat > $MAPPING
-wget "ftp://ftp.ncbi.nih.gov/genomes/H_sapiens/ARCHIVE/BUILD.37.2/mapview/seq_gene.md.gz" -O - | zcat > $MAPPING
-echo -e "${COLOR_INFO}Importing NCBI mapping data, this may take a few minutes (hg19)${COLOR_END}"
-$($BIN_MAPPING_UPDATE hg19 $MAPPING 'GRCh37.p2-Primary Assembly')
-rm $MAPPING
-
-echo -e "${COLOR_INFO}Separate import of NC_012920.1 (chrM) in hg19 database${COLOR_END}"
-$($BIN_MAPPING_IMPORT reference hg19 NC_012920.1)
-
-echo -e "${COLOR_INFO}Creating tables in mm10 database${COLOR_END}"
-
-# Create ChrName and Mapping table (mm10)
-cat << EOF | mysql -u mutalyzer -D mm10
-CREATE TABLE ChrName (
- AccNo char(20) NOT NULL,
- name char(20) NOT NULL,
- organelle_type enum('chromosome','mitochondrion') NOT NULL DEFAULT 'chromosome',
- PRIMARY KEY (AccNo)
-) ENGINE = MYISAM;
-CREATE TABLE Mapping (
- gene varchar(255) DEFAULT NULL,
- transcript varchar(20) NOT NULL DEFAULT '',
- version smallint(6) DEFAULT NULL,
- selector varchar(255) DEFAULT NULL,
- selector_version smallint(6) DEFAULT NULL,
- chromosome varchar(40) DEFAULT NULL,
- orientation char(1) DEFAULT NULL,
- start int(11) unsigned DEFAULT NULL,
- stop int(11) unsigned DEFAULT NULL,
- cds_start int(11) unsigned DEFAULT NULL,
- cds_stop int(11) unsigned DEFAULT NULL,
- exon_starts longblob NOT NULL,
- exon_stops longblob NOT NULL,
- protein varchar(20) DEFAULT NULL,
- source varchar(20) DEFAULT NULL,
- INDEX (transcript)
-) ENGINE = MYISAM;
-INSERT INTO ChrName (AccNo, name, organelle_type) VALUES
-('NC_000067.65', 'chr1', 'chromosome'),
-('NC_000068.70', 'chr2', 'chromosome'),
-('NC_000069.60', 'chr3', 'chromosome'),
-('NC_000070.66', 'chr4', 'chromosome'),
-('NC_000071.65', 'chr5', 'chromosome'),
-('NC_000072.60', 'chr6', 'chromosome'),
-('NC_000073.61', 'chr7', 'chromosome'),
-('NC_000074.60', 'chr8', 'chromosome'),
-('NC_000075.60', 'chr9', 'chromosome'),
-('NC_000076.60', 'chr10', 'chromosome'),
-('NC_000077.60', 'chr11', 'chromosome'),
-('NC_000078.60', 'chr12', 'chromosome'),
-('NC_000079.60', 'chr13', 'chromosome'),
-('NC_000080.60', 'chr14', 'chromosome'),
-('NC_000081.60', 'chr15', 'chromosome'),
-('NC_000082.60', 'chr16', 'chromosome'),
-('NC_000083.60', 'chr17', 'chromosome'),
-('NC_000084.60', 'chr18', 'chromosome'),
-('NC_000085.60', 'chr19', 'chromosome'),
-('NC_000086.71', 'chrX', 'chromosome'),
-('NC_000087.74', 'chrY', 'chromosome'),
-('NC_005089.1', 'chrM', 'mitochondrion');
-EOF
-
-echo -e "${COLOR_INFO}Populating Mapping table with NCBI data (mm10)${COLOR_END}"
-
-# Populate Mapping table with UCSC data (mm10)
-MAPPING=$(mktemp)
-wget "ftp://ftp.ncbi.nih.gov/genomes/M_musculus/ARCHIVE/BUILD.38.1/mapview/seq_gene.md.gz" -O - | zcat > $MAPPING
-echo -e "${COLOR_INFO}Importing NCBI mapping data, this may take a few minutes (mm10)${COLOR_END}"
-$($BIN_MAPPING_UPDATE mm10 $MAPPING 'GRCm38-C57BL/6J')
-rm $MAPPING
-
-echo -e "${COLOR_INFO}Separate import of NC_005089.1 (chrM) in mm10 database${COLOR_END}"
-$($BIN_MAPPING_IMPORT reference mm10 NC_005089.1)
-
-# The remainder is essentially the same as post-upgrade.sh
-
-if [ ! -e /var/www/mutalyzer ]; then
- mkdir -p /var/www/mutalyzer
-fi
-
-if [ -e /var/www/mutalyzer/base ]; then
- echo "Removing /var/www/mutalyzer/base"
- rm /var/www/mutalyzer/base
-fi
-
-echo -e "${COLOR_INFO}Symlinking /var/www/mutalyzer/base to $STATIC_DIR${COLOR_END}"
-ln -s $STATIC_DIR /var/www/mutalyzer/static
-
-echo "Restarting Apache"
-/etc/init.d/apache2 restart
-
-echo "Starting Mutalyzer batch daemon"
-/etc/init.d/mutalyzer-batchd start
diff --git a/extras/post-upgrade.sh b/extras/post-upgrade.sh
deleted file mode 100644
index d2264991dd09d11c6f9c0fa11d1ba54d496752c6..0000000000000000000000000000000000000000
--- a/extras/post-upgrade.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# Post-upgrade script for Mutalyzer. Run after the setuptools installation
-# (python setup.py install).
-#
-# Notice: The definitions in this file are quite specific to the standard
-# Mutalyzer environment. This consists of a Debian stable (Squeeze) system
-# with Apache and Mutalyzer using its mod_wsgi module. Debian conventions are
-# used throughout. See the README file for more information.
-#
-# Usage (from the source root directory):
-# sudo bash extras/post-upgrade.sh
-
-set -e
-set -u
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-# The 'cd /' is a hack to prevent the mutalyzer package under the current
-# directory to be used.
-STATIC_DIR=$(cd / && python -c 'import pkg_resources; print pkg_resources.resource_filename("mutalyzer", "templates/static")')
-BIN_WEBSITE=$(which mutalyzer-website.wsgi)
-BIN_SOAP_SERVICE=$(which mutalyzer-soap-service.wsgi)
-BIN_JSON_SERVICE=$(which mutalyzer-json-service.wsgi)
-
-if [ ! -e /var/www/mutalyzer ]; then
- mkdir -p /var/www/mutalyzer
-fi
-
-if [ -e /var/www/mutalyzer/base ]; then
- echo "Removing /var/www/mutalyzer/base"
- rm /var/www/mutalyzer/base
-fi
-
-echo -e "${COLOR_INFO}Symlinking /var/www/mutalyzer/base to $STATIC_DIR${COLOR_END}"
-ln -s $STATIC_DIR /var/www/mutalyzer/static
-
-echo "Running any needed migrations"
-for MIGRATION in extras/migrations/*.migration; do
- echo "Checking migration $(basename $MIGRATION)"
- $MIGRATION migrate
-done
-
-echo -e "${COLOR_INFO}Assuming mod_wsgi daemon mode, not restarting Apache${COLOR_END}"
-#/etc/init.d/apache2 restart
-
-echo "Touching WSGI entry to reload application"
-touch $BIN_WEBSITE
-touch $BIN_SOAP_SERVICE
-touch $BIN_JSON_SERVICE
-
-echo "Restarting Mutalyzer batch daemon"
-/etc/init.d/mutalyzer-batchd restart
diff --git a/extras/pre-install.sh b/extras/pre-install.sh
deleted file mode 100644
index 8ef26d587034fe4875f4bab637ff40407fa3a97c..0000000000000000000000000000000000000000
--- a/extras/pre-install.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-
-# Pre-install script for Mutalyzer on Debian or Debian-like systems. Run
-# before the setuptools installation (python setup.py install).
-#
-# Notice: The definitions in this file are quite specific to the standard
-# Mutalyzer environment. This consists of a Debian stable (Squeeze) system
-# with Apache and Mutalyzer using its mod_wsgi module. Debian conventions are
-# used throughout. See the README file for more information.
-#
-# Usage (from the source root directory):
-# sudo bash extras/pre-install.sh
-
-set -e
-set -u
-
-COLOR_INFO='\033[32m'
-COLOR_WARNING='\033[33m'
-COLOR_ERROR='\033[31m'
-COLOR_END='\033[0m'
-
-echo -e "${COLOR_INFO}Installing packages with apt${COLOR_END}"
-
-apt-get update && \
-apt-get install -y \
- mysql-server \
- python \
- python-mysqldb \
- python-biopython \
- python-pyparsing \
- python-configobj \
- python-simpletal \
- python-soappy \
- python-suds \
- python-magic \
- python-psutil \
- python-xlrd \
- python-daemon \
- python-webpy \
- python-webtest \
- python-nose \
- python-werkzeug \
- apache2 \
- libapache2-mod-wsgi \
- python-setuptools \
- git-core
-
-echo -e "${COLOR_INFO}Installing known-working spyne from git${COLOR_END}"
-
-# For now we use a specific known-working version of spyne.
-pushd $(mktemp -d)
-git clone https://github.com/LUMC/spyne.git .
-git checkout mutalyzer
-python setup.py install
-popd
-
-echo -e "${COLOR_INFO}Installing suds using easy_install${COLOR_END}"
-
-easy_install suds
diff --git a/fabfile.py b/fabfile.py
deleted file mode 100644
index 9f6f8542abffe9e9427c42b558ba622508fc2f25..0000000000000000000000000000000000000000
--- a/fabfile.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-Fabric fabfile for Mutalyzer.
-
-Notice: The definitions in this file are quite specific to the standard
-Mutalyzer environment. This consists of a Debian stable (Squeeze) system with
-Apache and Mutalyzer using its mod_wsgi module. Debian conventions are used
-throughout. See the README file for more information.
-
-To do a deployment on a server with an existing configured Mutalyzer
-installation:
-
- $ fab deploy -H server1.mutalyzer.nl
-
-For a fresh deployment on a new server:
-
- $ fab deploy:boostrap=yes -H server1.mutalyzer.nl
-"""
-
-
-from fabric.api import *
-
-
-def deploy(bootstrap='no'):
- """
- Deploy Mutalyzer on the remote host.
-
- Create a source distribution, transfer it to the remote host, and install
- from there. After installation, we restart Apache and the Mutalyzer batch
- daemon.
-
- Additionally, if bootstrap=yes, install all dependencies before Mutalyzer
- installation, and bootstrap the Mutalyzer configuration afterwards (i.e.
- create and fill database, add cron script, create cache directory, etc).
- """
- # Currently, Fabric only supports task arguments as strings.
- bootstrap = (bootstrap == 'yes')
-
- # Create a new source distribution as a tarball.
- local('python setup.py sdist --formats=gztar')
-
- # Figure out the release name and tarball filename.
- dist = local('python setup.py --fullname', capture=True).strip()
- tarball = '%s.tar.gz' % dist
-
- # Create a place where we can unzip the source tarball.
- tempdir = run('mktemp -d').strip()
-
- # Upload the source tarball to the temporary folder on the server.
- put('dist/%s' % tarball, tempdir + '/%s' % tarball)
-
- # Go to that directory, unzip and install it.
- with cd(tempdir):
- run('tar xzf %s' % tarball)
-
- # Go to the tarball's contents and do the installation.
- with cd(tempdir + '/%s' % dist):
-
- if bootstrap:
- # Install dependencies.
- sudo('bash extras/pre-install.sh')
-
- # Install Mutalyzer.
- sudo('python setup.py install')
-
- if bootstrap:
- # Configure Mutalyzer.
- sudo('bash extras/post-install.sh')
- else:
- # Restart services.
- sudo('bash extras/post-upgrade.sh')
-
- # Run unittests.
- #run('nosetests -v')
-
- # Now that all is set up, delete the folder again.
- # (I don't like to 'sudo rm -Rf'.)
- #sudo('rm -Rf %s' % tempdir)
diff --git a/mutalyzer/config/__init__.py b/mutalyzer/config/__init__.py
index ffc400625cee228276945a9071ea06159ccc16e4..def4630bc53ad26234896ab56165afa73bdc3c88 100644
--- a/mutalyzer/config/__init__.py
+++ b/mutalyzer/config/__init__.py
@@ -20,6 +20,7 @@ import flask.config
from mutalyzer import util
+#: Environment variable used for locating the settings module.
ENVIRONMENT_VARIABLE = 'MUTALYZER_SETTINGS'
@@ -104,4 +105,6 @@ class LazySettings(util.LazyObject):
self._callbacks[key].append(callback)
+#: Global :class:`LazySettings` instance. Use this for querying configuration
+#: settings.
settings = LazySettings()
diff --git a/mutalyzer/db/__init__.py b/mutalyzer/db/__init__.py
index 3e4fc6ea9b8ae333977ac99e29ffb33eccc69a5e..b2192186773b542c68d1dae3884124f012ccfff4 100644
--- a/mutalyzer/db/__init__.py
+++ b/mutalyzer/db/__init__.py
@@ -66,11 +66,17 @@ def configure_session(uri):
settings.on_update(configure_session, 'DATABASE_URI')
-# Session are automatically created where needed and are scoped by thread.
+# Sessions are automatically created where needed and are scoped by thread.
session_factory = SessionFactory()
+
+#: Global scoped :class:`sqlalchemy.orm.session.Session` instance. Use this
+#: for all database communication, except for querying models. For the latter,
+#: each model has a `query` property that is a
+#: :class:`sqlalchemy.orm.query.Query` object against the model and the
+#: current `Session` when called.
session = scoped_session(session_factory)
-# Base class to use in our models.
+#: Base class to use for our models.
Base = declarative_base()
Base.query = session.query_property()
diff --git a/mutalyzer/entrypoints/admin.py b/mutalyzer/entrypoints/admin.py
index c48b96d39ab4df782dcb162f44d2c5cdc982f215..329e13eaa5b20551af4d35e38bcfc3a5cb275d1c 100644
--- a/mutalyzer/entrypoints/admin.py
+++ b/mutalyzer/entrypoints/admin.py
@@ -130,6 +130,8 @@ def sync_cache(wsdl_url, url_template, history=7):
This program is intended to be run daily from cron. Example:
+ .. code-block:: none
+
25 5 * * * mutalyzer-cache-sync 'http://dom1/?wsdl' 'http://dom1/{file}' -H 7
55 5 * * * mutalyzer-cache-sync 'http://dom2/?wsdl' 'http://dom2/{file}' -H 7
"""
diff --git a/mutalyzer/entrypoints/batch_processor.py b/mutalyzer/entrypoints/batch_processor.py
index ff2ac02bc8d668b48e56582b6ec4344e06ee9367..fa684b59ca7fc8520ae6bced6b3b19bf388cece8 100644
--- a/mutalyzer/entrypoints/batch_processor.py
+++ b/mutalyzer/entrypoints/batch_processor.py
@@ -23,11 +23,11 @@ def process():
def handle_exit(signum, stack_frame):
if scheduler.stopped():
- sys.stderr.write('mutalyzer-batchd: Terminated\n')
+ sys.stderr.write('mutalyzer-batch-processor: Terminated\n')
sys.exit(1)
if signum == signal.SIGINT:
- sys.stderr.write('mutalyzer-batchd: Hitting Ctrl+C again will '
- 'terminate any running job!\n')
+ sys.stderr.write('mutalyzer-batch-processor: Hitting Ctrl+C '
+ 'again will terminate any running job!\n')
scheduler.stop()
signal.signal(signal.SIGTERM, handle_exit)
@@ -41,7 +41,7 @@ def process():
# Wait a bit and process any possible new jobs.
time.sleep(1)
- sys.stderr.write('mutalyzer-batchd: Graceful shutdown\n')
+ sys.stderr.write('mutalyzer-batch-processor: Graceful shutdown\n')
sys.exit(0)
diff --git a/mutalyzer/entrypoints/service_json.py b/mutalyzer/entrypoints/service_json.py
index 9dd1952ecbd3d35c9972464e7e40e8bad127158a..fb2e0c2c0263219dd19d5cd20e5646664681d1c8 100644
--- a/mutalyzer/entrypoints/service_json.py
+++ b/mutalyzer/entrypoints/service_json.py
@@ -3,10 +3,14 @@ WSGI interface to the Mutalyzer HTTP/RPC+JSON webservice.
Example Apache/mod_wsgi configuration:
+.. code-block:: apache
+
WSGIScriptAlias /json /usr/local/bin/mutalyzer-service-json
Be sure to have this line first if you also define a / alias, like this:
+.. code-block:: apache
+
WSGIScriptAlias /json /usr/local/bin/mutalyzer-service-json
WSGIScriptAlias / /usr/local/bin/mutalyzer-website
@@ -23,6 +27,7 @@ from spyne.server.wsgi import WsgiApplication
from ..services import json
+#: WSGI application instance.
application = WsgiApplication(json.application)
diff --git a/mutalyzer/entrypoints/service_soap.py b/mutalyzer/entrypoints/service_soap.py
index eb91a82e8d5ce4aa7d4a4530e9577f78be02d3bb..3728bdf1f9c4141ff36b78299d9d0b1c3f9241b2 100644
--- a/mutalyzer/entrypoints/service_soap.py
+++ b/mutalyzer/entrypoints/service_soap.py
@@ -3,10 +3,14 @@ WSGI interface to the Mutalyzer SOAP webservice.
Example Apache/mod_wsgi configuration:
+.. code-block:: apache
+
WSGIScriptAlias /soap /usr/local/bin/mutalyzer-service-soap
Be sure to have this line first if you also define a / alias, like this:
+.. code-block:: apache
+
WSGIScriptAlias /soap /usr/local/bin/mutalyzer-service-soap
WSGIScriptAlias / /usr/local/bin/mutalyzer-website
@@ -23,6 +27,7 @@ from spyne.server.wsgi import WsgiApplication
from ..services import soap
+#: WSGI application instance.
application = WsgiApplication(soap.application)
diff --git a/mutalyzer/entrypoints/website.py b/mutalyzer/entrypoints/website.py
index 457422d792338ac343bc5858aa1e7b9f5c70f32e..a9faccc3e59008ba6e17f580b00705aa6d64ac33 100644
--- a/mutalyzer/entrypoints/website.py
+++ b/mutalyzer/entrypoints/website.py
@@ -1,11 +1,13 @@
"""
WSGI interface to the Mutalyzer website.
-The WSGI interface is exposed through the module variable 'application'.
+The WSGI interface is exposed through the module variable :data:`application`.
Static files are not handled by this interface and should be served through
-the '/static' url prefix separately.
+the ``/static`` url prefix separately.
-Example Apache/mod_wsgi configuration:
+Example *Apache/mod_wsgi* configuration:
+
+.. code-block:: apache
Alias /static /var/www/mutalyzer/static
WSGIScriptAlias / /usr/local/bin/mutalyzer-website
@@ -15,6 +17,8 @@ and act as a reverse proxy server to the Mutalyzer HTTP server.
Example Nginx configuration:
+.. code-block:: nginx
+
server {
listen 80;
location /static/ {
@@ -40,6 +44,7 @@ import argparse
from .. import website
+#: WSGI application instance.
application = website.create_app()
diff --git a/mutalyzer/redisclient.py b/mutalyzer/redisclient.py
index 8b3138a8018ca3f5152cd6f1abf91b6d85f36415..ec9e6050548a85d04dced7489fbd8de195a5c6fc 100644
--- a/mutalyzer/redisclient.py
+++ b/mutalyzer/redisclient.py
@@ -40,4 +40,6 @@ class LazyClient(util.LazyObject):
self._wrapped = redis.StrictRedis.from_url(settings.REDIS_URI)
+#: Global :class:`LazyClient` instance. Use this for all communication with
+#: Redis.
client = LazyClient()
diff --git a/mutalyzer/services/json.py b/mutalyzer/services/json.py
index 6362282c7b40f4d9935b0424aa6c8c96e02eb97a..c35b79293c1a790209185a9efa37772155acf07e 100644
--- a/mutalyzer/services/json.py
+++ b/mutalyzer/services/json.py
@@ -11,7 +11,7 @@ import mutalyzer
from mutalyzer.services import rpc
-# HTTP/RPC application.
+#: HTTP/RPC+JSON application.
application = Application([rpc.MutalyzerService], tns=mutalyzer.SOAP_NAMESPACE,
in_protocol=HttpRpc(validator='soft'),
out_protocol=JsonDocument(),
diff --git a/mutalyzer/services/soap.py b/mutalyzer/services/soap.py
index 5674e6d240e7468269c55acb1cb66b62323ce1e3..a7d7b001868705b65807f27edba653c1050e6fda 100644
--- a/mutalyzer/services/soap.py
+++ b/mutalyzer/services/soap.py
@@ -10,7 +10,7 @@ import mutalyzer
from mutalyzer.services import rpc
-# SOAP/1.1 application.
+#: SOAP/1.1 application.
application = Application([rpc.MutalyzerService], tns=mutalyzer.SOAP_NAMESPACE,
in_protocol=Soap11(validator='lxml'),
out_protocol=Soap11(),
diff --git a/mutalyzer/sync.py b/mutalyzer/sync.py
index 40c64c57212be8b7cbcfdd5aa79025e8d31820b3..4c278e819d87d0d64f793057444659f25c1c3a3e 100644
--- a/mutalyzer/sync.py
+++ b/mutalyzer/sync.py
@@ -1,5 +1,5 @@
"""
-Module for synchronizing the database with other Mutalyzer instances.
+Synchronizing the reference file cache with other Mutalyzer instances.
"""
@@ -19,6 +19,7 @@ from mutalyzer.db.models import Reference
from mutalyzer import Retriever
+#: Default number of days to go back in synchronization.
DEFAULT_CREATED_SINCE_DAYS = 7
@@ -30,22 +31,22 @@ class CacheSync(object):
"""
Instantiate the object.
- @arg output: An output object.
- @type output: mutalyzer.output.Output
+ :arg output: An output object.
+ :type output: :class:`mutalyzer.output.Output`
"""
self._output = output
def local_cache(self, created_since=None):
"""
- Get all entries in the local cache with creation date {created_since}
+ Get all entries in the local cache with creation date `created_since`
or later.
- @kwarg created_since: Only entries with this creation date or later
- are returned.
- @type created_since: datatime.datetime
+ :arg created_since: Only entries with this creation date or later
+ are returned.
+ :type created_since: datatime.datetime
- @return: List of cache entries.
- @rtype: list(dictionary)
+ :return: List of cache entries.
+ :rtype: list(dict)
"""
if not created_since:
created_since = datetime.today() - \
@@ -81,17 +82,17 @@ class CacheSync(object):
def remote_cache(self, remote_wsdl, created_since=None):
"""
- Get all entries in the remote cache with creation date {created_since}
+ Get all entries in the remote cache with creation date `created_since`
or later.
- @arg remote_wsdl: The url of the remote SOAP WSDL description.
- @type remote_wsdl: string
- @kwarg created_since: Only entries with this creation date or later
- are returned.
- @type created_since: datatime.datetime
+ :arg remote_wsdl: The url of the remote SOAP WSDL description.
+ :type remote_wsdl: str
+ :arg created_since: Only entries with this creation date or later
+ are returned.
+ :type created_since: datatime.datetime
- @return: List of cache entries.
- @rtype: list(dictionary)
+ :return: List of cache entries.
+ :rtype: list(dict)
"""
self._output.addMessage(__file__, -1, 'INFO', 'Getting remote cache'
' from %s' % remote_wsdl)
@@ -122,12 +123,12 @@ class CacheSync(object):
def store_remote_file(self, name, url):
"""
- Download a remote file located at {url} and store it as {name}.
+ Download a remote file located at `url` and store it as `name`.
- @arg name: Name to store the file under.
- @type name: string
- @arg url: Url to the remote file.
- @type url: string
+ :arg name: Name to store the file under.
+ :type name: str
+ :arg url: Url to the remote file.
+ :type url: str
"""
if not re.match('^[\da-zA-Z\._-]+$', name):
return
@@ -146,23 +147,25 @@ class CacheSync(object):
"""
Synchronize the local cache with the remote cache.
+ ::
+
>>> wsdl = 'http://mutalyzer.nl/mutalyzer/services/?wsdl'
>>> template = 'http://mutalyzer.nl/mutalyzer/Reference/{file}'
>>> self.sync_with_remote(wsdl, template)
(14, 3)
- @arg remote_wsdl: The url of the remote SOAP WSDL description.
- @type remote_wsdl: string
- @arg url_template: Formatting string containing a {file} occurence,
- see examle usage above.
- @string url_template: string
- @kwarg days: Only remote entries added this number of days ago or
- later are considered.
- @type days: int
-
- @return: The number of entries added to the local cache and the number
- cache files downloaded from the remote site.
- @rtype: tuple(int, int)
+ :arg remote_wsdl: The url of the remote SOAP WSDL description.
+ :type remote_wsdl: str
+ :arg url_template: Formatting string containing a ``{file}``
+ occurence, see example usage above.
+ :string url_template: str
+ :arg days: Only remote entries added this number of days ago or
+ later are considered.
+ :type days: int
+
+ :return: The number of entries added to the local cache and the number
+ cache files downloaded from the remote site.
+ :rtype: tuple(int, int)
"""
self._output.addMessage(__file__, -1, 'INFO', 'Starting cache sync')
diff --git a/mutalyzer/website/views.py b/mutalyzer/website/views.py
index 98f79ed986b6308f57b3813f0d2783a224bdb6d9..510719c503d8ff55913506e38b9ee7c542c397f2 100644
--- a/mutalyzer/website/views.py
+++ b/mutalyzer/website/views.py
@@ -880,17 +880,20 @@ def lovd_get_gs():
LOVD bypass to get the correct GeneSymbol incl Transcript variant.
Used by LOVD to get the correct transcript variant out of a genomic
- record. LOVD uses a genomic reference (NC_?) in combination with a gene
- symbol to pass variant info to mutalyzer. Mutalyzer 1.0 was only using
- the first transcript. LOVD supplies the NM of the transcript needed but
- this was ignored. This helper allows LOVD to get the requested
+ record. LOVD uses a genomic reference (``NC_``?) in combination with a
+ gene symbol to pass variant info to mutalyzer. Mutalyzer 1.0 was only
+ using the first transcript. LOVD supplies the NM of the transcript needed
+ but this was ignored. This helper allows LOVD to get the requested
transcript variant from a genomic reference.
Parameters:
- - `mutationName`: The mutationname without gene symbol.
- - `variantRecord`: The NM reference of the variant.
- - `forward`: If set this forwards the request to the name checker.
+ mutationName
+ The mutationname without gene symbol.
+ variantRecord
+ The NM reference of the variant.
+ forward
+ If set this forwards the request to the name checker.
Returns: Output of name checker if `forward` is set, otherwise the
gene symbol with the variant notation as string.
@@ -960,28 +963,42 @@ def lovd_variant_info():
Parameters:
- - `LOVD_ver`: The version of the calling LOVD.
- - `build`: The human genome build (hg19 assumed).
- - `acc`: The accession number (NM number).
- - `var`: A description of the variant.
+ LOVD_ver
+ The version of the calling LOVD.
+ build
+ The human genome build (hg19 assumed).
+ acc
+ The accession number (NM number).
+ var
+ A description of the variant.
Returns:
- - start_main ; The main coordinate of the start position in I{c.}
- (non-star) notation.
- - start_offset ; The offset coordinate of the start position in I{c.}
- notation (intronic position).
- - end_main ; The main coordinate of the end position in I{c.}
- (non-star) notation.
- - end_offset ; The offset coordinate of the end position in I{c.}
- notation (intronic position).
- - start_g ; The I{g.} notation of the start position.
- - end_g ; The I{g.} notation of the end position.
- - type ; The mutation type.
+
+ start_main
+ The main coordinate of the start position in I{c.} (non-star) notation.
+ start_offset
+ The offset coordinate of the start position in I{c.} notation (intronic
+ position).
+ end_main
+ The main coordinate of the end position in I{c.} (non-star) notation.
+ end_offset
+ The offset coordinate of the end position in I{c.} notation (intronic
+ position).
+ start_g
+ The I{g.} notation of the start position.
+ end_g
+ The I{g.} notation of the end position.
+ type
+ The mutation type.
Returns (alternative):
- - trans_start ; Transcription start in I{c.} notation.
- - trans_stop ; Transcription stop in I{c.} notation.
- - CDS_stop ; CDS stop in I{c.} notation.
+
+ trans_start
+ Transcription start in I{c.} notation.
+ trans_stop
+ Transcription stop in I{c.} notation.
+ CDS_stop
+ CDS stop in I{c.} notation.
"""
lovd_version = request.args['LOVD_ver']
build = request.args['build']
diff --git a/requirements.txt b/requirements.txt
index 048f622863eefc358fdfb74a858b27516eea0ad3..1add561a8523a44684d66843cacf24660ac079be 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,3 +19,5 @@ redis==2.9.1
mockredispy==2.9.0.1
mock==1.0.1
alembic==0.6.3
+Sphinx==1.2.1
+sphinx-rtd-theme==0.1.5
diff --git a/setup.py b/setup.py
index 8e4ce82a1e4896cb6ad50255b247af617be74094..c9d7e0916b251081c7b3f9058df60ff620f5b6c9 100644
--- a/setup.py
+++ b/setup.py
@@ -2,13 +2,13 @@ import os
import sys
from setuptools import setup
-if sys.version_info < (2, 6):
- raise Exception('Mutalyzer requires Python 2.6 or higher.')
+if sys.version_info < (2, 7):
+ raise Exception('Mutalyzer requires Python 2.7 or higher.')
install_requires = []
try:
- with open('README.md') as readme:
+ with open('README.rst') as readme:
long_description = readme.read()
except IOError:
long_description = 'See https://mutalyzer.nl'
@@ -57,11 +57,3 @@ setup(
'mutalyzer-website = mutalyzer.entrypoints.website:main']},
zip_safe=False
)
-
-# Things not handled by this setup.py:
-# - Copy extras/config.example to /etc/mutalyzer/config
-# - Database setup
-# - Chown /var/log/mutalyzer.log and /var/cache/mutalyzer
-# - Copy extras/init.d/mutalyzer-batchd to /etc/init.d/mutalyzer-batchd
-# - Copy doc to /usr/share/doc
-# Check extras/post-install.sh for these.