Skeleton for cache sync. Suds monkey patch in mutalyzer.util.

git-svn-id: https://humgenprojects.lumc.nl/svn/mutalyzer/branches/gbinfo-sync-branch@314 eb6bd6ab-9ccd-42b9-aceb-e2899b4a52f1

bin/mutalyzer-cache-sync

@@ -11,6 +11,8 @@ This program is intended to be run daily from cron. Example:
 
 from mutalyzer.config import Config
 from mutalyzer.output import Output
+from mutalyzer.sync import CacheSync
+from mutalyzer import Db
 
 
 def main():
@@ -22,7 +24,11 @@ def main():
     output.addMessage(__file__, -1, 'INFO',
                       'Starting cache sync')
 
-    # Todo: Actually do something.
+    database = Db.Cache(config.Db)
+    sync = CacheSync(config.Sync, database)
+
+    created_since = datetime.today() - timedelta(days=60)
+    sync.sync_with_remote(created_since)
 
     output.addMessage(__file__, -1, 'INFO', 'Cache sync end')
 

extras/soap-tools/getcache.py

@@ -25,7 +25,7 @@ days = 1
 if len(sys.argv) > 1:
     days = int(sys.argv[1])
 
-created_since = datetime.today() - timedelta(minutes=days)
+created_since = datetime.today() - timedelta(days=days)
 
 print 'Getting cache...'
 

mutalyzer/Db.py

@@ -890,6 +890,7 @@ class Cache(Db) :
 
         statement = """
             INSERT INTO GBInfo
+             (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url)
               VALUES (%s, %s, %s, %s, %s, %s, %s, %s);
         """, (accNo, GI, fileHash, ChrAccVer, ChrStart, ChrStop, orientation,
               url)
@@ -913,6 +914,7 @@ class Cache(Db) :
 
         statement = """
             INSERT INTO GBInfo
+             (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url)
               VALUES (%s, %s, %s, %s, %s, %s, %s, %s);
         """, (accNo, None, fileHash, None, None, None, None, url)
 
@@ -1048,7 +1050,8 @@ class Cache(Db) :
         @rtype: string
         """
         statement = """
-            SELECT *
+            SELECT
+            (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url, created)
             FROM GBInfo
             WHERE created >= %s;
         """, created_since

mutalyzer/sync.py

@@ -3,9 +3,14 @@ Module for synchronizing the database with other Mutalyzer instances.
 """
 
 
+from mutalyzer.util import monkey_patch_suds; monkey_patch_suds()
+from suds.client import Client
 from datetime import datetime, timedelta
 
 
+DEFAULT_CREATED_SINCE_DAYS = 7
+
+
 class CacheSync(object):
     """
     Todo.
@@ -22,5 +27,54 @@ class CacheSync(object):
         Todo.
         """
         if not created_since:
-            created_since = datetime.today() - timedelta(days=7)
+            created_since = datetime.today() - \
+                            timedelta(days=DEFAULT_CREATED_SINCE_DAYS)
         return self._database.getGB(created_since)
+
+    def remote_cache(self, remote_wsdl, created_since=None):
+        """
+        Todo.
+        """
+        if not created_since:
+            created_since = datetime.today() - \
+                            timedelta(days=DEFAULT_CREATED_SINCE_DAYS)
+        client = Client(remote_wsdl, cache=None)
+        cache = client.service.getCache(created_since)
+
+        def cache_entry_from_soap(entry):
+            """
+            Create a nice dictionary out of the CacheEntry object.
+            """
+            entry_dict =  {'name':    entry.name,
+                           'hash':    entry.hash,
+                           'created': entry.created}
+            for attribute in ('gi', 'chromosomeName', 'chromosomeStart'
+                              'chromosomeStop', 'chromosomeOrientation',
+                              'url'):
+                entry_dict[attribute] = entry[attribute] \
+                                        if attribute in entry else None
+            return entry_dict
+
+        return map(cache_entry_from_soap, cache.CacheEntry)
+
+    def sync_with_remote(self, remote_wsdl, created_since=None):
+        """
+        Todo.
+        """
+        remote_cache = self.remote_cache(remote_wsdl, created_since)
+
+        for entry in remote_cache:
+            if self._database.getHash(entry['name']):
+                continue
+            #self._database.insertGB(entry['name'],
+            #                        entry['gi'],
+            #                        entry['hash'],
+            #                        entry['chromosomeName'],
+            #                        entry['chromosomeStart'],
+            #                        entry['chromosomeStop'],
+            #                        entry['chromosomeOrientation'],
+            #                        entry['url'])
+            print 'inserting %s' % entry['name']
+            print entry
+            if not entry['chromosomeName'] and not entry['url']:
+                print '(downloading file from remote cache...)'

mutalyzer/util.py

@@ -739,3 +739,30 @@ def slow(f):
             f(*args, **kwargs)
     return slow_f
 #slow
+
+
+def monkey_patch_suds():
+    """
+    Apply our monkey-patch for the suds package.
+
+    For some weird reason the location http://www.w3.org/2001/xml.xsd is used
+    for the XML namespace, but the W3C seems to respond too slow on that url.
+    We therefore use http://www.w3.org/2009/01/xml.xsd which fixes this.
+
+    Call this function before importing anything from the suds package.
+    """
+    from suds.xsd.sxbasic import Import
+    _import_open = Import.open
+
+    # Only apply the patch once.
+    if getattr(Import, 'MUTALYZER_MONKEY_PATCHED', False):
+        return
+
+    def _import_open_patched(self, *args, **kwargs):
+        if self.location == 'http://www.w3.org/2001/xml.xsd':
+            self.location = 'http://www.w3.org/2009/01/xml.xsd'
+        return _import_open(self, *args, **kwargs)
+
+    Import.open = _import_open_patched
+    Import.MUTALYZER_MONKEY_PATCHED = True
+#monkey_patch_suds

mutalyzer/webservice.py

@@ -897,23 +897,23 @@ class MutalyzerService(DefinitionBase):
 
         cache = sync.local_cache(created_since)
 
-        def soap_cache_entry(entry):
+        def cache_entry_to_soap(entry):
             e = CacheEntry()
-            e.name = entry[0]
-            e.gi = entry[1]
-            e.hash = entry[2]
-            e.chromosomeName = entry[3]
-            e.chromosomeStart = entry[4]
-            e.chromosomeStop = entry[5]
-            e.chromosomeOrientation = entry[6]
-            e.url = entry[7]
-            e.created = entry[8]
+            (e.name,
+             e.gi,
+             e.hash,
+             e.chromosomeName,
+             e.chromosomeStart,
+             e.chromosomeStop,
+             e.chromosomeOrientation,
+             e.url,
+             e.created) = entry
             return e
 
         output.addMessage(__file__, -1, 'INFO',
                           'Finished processing getCache')
 
-        return map(soap_cache_entry, cache)
+        return map(cache_entry_to_soap, cache)
     #getCache
 #MutalyzerService
 

tests/test_webservice.py

@@ -3,21 +3,14 @@ Tests for the SOAP interface to Mutalyzer.
 """
 
 
-# Monkey patch suds, because for some weird reason the location
-# http://www.w3.org/2001/xml.xsd is used for the XML namespace, but the W3C
-# seems to respond too slow on that url. We use therefore use
-# http://www.w3.org/2009/01/xml.xsd which fixes this.
-from suds.xsd.sxbasic import Import
-_import_open = Import.open
-def _import_open_patched(self, *args, **kwargs):
-    if self.location == 'http://www.w3.org/2001/xml.xsd':
-        self.location = 'http://www.w3.org/2009/01/xml.xsd'
-    return _import_open(self, *args, **kwargs)
-Import.open = _import_open_patched
-
+from mutalyzer.util import monkey_patch_suds; monkey_patch_suds()
 
 import os
+from datetime import datetime, timedelta
 import mutalyzer
+from mutalyzer.config import Config
+from mutalyzer.sync import CacheSync
+from mutalyzer import Db
 import logging; logging.raiseExceptions = 0
 import urllib2
 from suds.client import Client
@@ -169,3 +162,17 @@ class TestWebservice():
         r = self.client.service.info()
         assert_equal(type(r.versionParts.string), list)
         assert_equal(r.version, mutalyzer.__version__)
+
+    def test_getcache(self):
+        """
+        Running the getCache method should give us the expected cache entries.
+        """
+        created_since = datetime.today() - timedelta(days=60)
+
+        config = Config()
+        database = Db.Cache(config.Db)
+        sync = CacheSync(config.Sync, database)
+        cache = sync.local_cache(created_since)
+
+        r = self.client.service.getCache(created_since)
+        assert_equal(len(r.CacheEntry), len(cache))