Merge pull request #52 from mutalyzer/cchardet-chardet

Use chardet instead of cchardet

doc/conf.py

@@ -23,8 +23,7 @@ class Mock(MagicMock):
     def __getattr__(cls, name):
         return Mock()
 
-MOCK_MODULES = ['MySQLdb', 'cchardet', 'lxml', 'lxml.builder', 'lxml.etree',
-                'magic']
+MOCK_MODULES = ['MySQLdb', 'lxml', 'lxml.builder', 'lxml.etree', 'magic']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 # If extensions (or modules to document with autodoc) are in another directory,

mutalyzer/File.py

@@ -25,7 +25,7 @@ import csv             # Sniffer(), reader(), Error
 import xlrd            # open_workbook()
 import zipfile         # ZipFile()
 import xml.dom.minidom # parseString()
-import cchardet as chardet
+import chardet
 
 from mutalyzer.config import settings
 

mutalyzer/Retriever.py

@@ -27,7 +27,7 @@ from xml.dom import DOMException, minidom
 from xml.parsers import expat
 from httplib import HTTPException, IncompleteRead
 from sqlalchemy.orm.exc import NoResultFound
-import cchardet as chardet
+import chardet
 
 from mutalyzer import util
 from mutalyzer.config import settings

requirements.txt

@@ -8,7 +8,7 @@ Sphinx==1.2.3
 Werkzeug==0.9.6
 alembic==0.6.7
 biopython==1.64
-cchardet==0.3.5
+chardet==2.3.0
 cssselect==0.9.1
 lxml==3.4.0
 mock==1.0.1

tests/test_scheduler.py

@@ -350,3 +350,21 @@ class TestScheduler(MutalyzerTest):
                     ['\u2026AL449423.14(CDKN2A_v002):c.5_400del',
                      '(grammar): Expected W:(0123...) (at char 0), (line:1, col:1)']]
         self._batch_job_plain_text(variants, expected, 'syntax-checker')
+
+    def test_windows_1252_input(self):
+        """
+        Simple input encoded as WINDOWS-1252.
+        """
+        variants = ['AB026906.1:c.274G>T',
+                    # Encoded as WINDOWS-1252, the following is not valid UTF8.
+                    'NM_000052.4:c.2407\u20132A>G',
+                    'AL449423.14(CDKN2A_v002):c.5_400del']
+        batch_file = io.BytesIO(('\n'.join(variants) + '\n').encode('WINDOWS-1252'))
+        expected = [['AB026906.1:c.274G>T',
+                     'OK'],
+                    ['NM_000052.4:c.2407\u20132A>G',
+                     '(grammar): Expected W:(acgt...) (at char 18), (line:1, col:19)'],
+                    ['AL449423.14(CDKN2A_v002):c.5_400del',
+                     'OK']]
+
+        self._batch_job(batch_file, expected, 'syntax-checker')