From 674d94c1950e1f3189a12c9a537c0dac3716181a Mon Sep 17 00:00:00 2001
From: Martijn Vermaat <martijn@vermaat.name>
Date: Mon, 25 Jul 2011 14:37:12 +0000
Subject: [PATCH] Skeleton for cache sync. Suds monkey patch in mutalyzer.util.
git-svn-id: https://humgenprojects.lumc.nl/svn/mutalyzer/branches/gbinfo-sync-branch@314 eb6bd6ab-9ccd-42b9-aceb-e2899b4a52f1
---
bin/mutalyzer-cache-sync | 8 ++++-
extras/soap-tools/getcache.py | 2 +-
mutalyzer/Db.py | 5 +++-
mutalyzer/sync.py | 56 ++++++++++++++++++++++++++++++++++-
mutalyzer/util.py | 27 +++++++++++++++++
mutalyzer/webservice.py | 22 +++++++-------
tests/test_webservice.py | 31 +++++++++++--------
7 files changed, 124 insertions(+), 27 deletions(-)
diff --git a/bin/mutalyzer-cache-sync b/bin/mutalyzer-cache-sync
index 2a276aab..a58d51ad 100755
--- a/bin/mutalyzer-cache-sync
+++ b/bin/mutalyzer-cache-sync
@@ -11,6 +11,8 @@ This program is intended to be run daily from cron. Example:
from mutalyzer.config import Config
from mutalyzer.output import Output
+from mutalyzer.sync import CacheSync
+from mutalyzer import Db
def main():
@@ -22,7 +24,11 @@ def main():
output.addMessage(__file__, -1, 'INFO',
'Starting cache sync')
- # Todo: Actually do something.
+ database = Db.Cache(config.Db)
+ sync = CacheSync(config.Sync, database)
+
+ created_since = datetime.today() - timedelta(days=60)
+ sync.sync_with_remote(created_since)
output.addMessage(__file__, -1, 'INFO', 'Cache sync end')
diff --git a/extras/soap-tools/getcache.py b/extras/soap-tools/getcache.py
index fc8cd298..cf9baa9e 100755
--- a/extras/soap-tools/getcache.py
+++ b/extras/soap-tools/getcache.py
@@ -25,7 +25,7 @@ days = 1
if len(sys.argv) > 1:
days = int(sys.argv[1])
-created_since = datetime.today() - timedelta(minutes=days)
+created_since = datetime.today() - timedelta(days=days)
print 'Getting cache...'
diff --git a/mutalyzer/Db.py b/mutalyzer/Db.py
index ce7b8e32..ea559584 100644
--- a/mutalyzer/Db.py
+++ b/mutalyzer/Db.py
@@ -890,6 +890,7 @@ class Cache(Db) :
statement = """
INSERT INTO GBInfo
+ (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s);
""", (accNo, GI, fileHash, ChrAccVer, ChrStart, ChrStop, orientation,
url)
@@ -913,6 +914,7 @@ class Cache(Db) :
statement = """
INSERT INTO GBInfo
+ (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s);
""", (accNo, None, fileHash, None, None, None, None, url)
@@ -1048,7 +1050,8 @@ class Cache(Db) :
@rtype: string
"""
statement = """
- SELECT *
+ SELECT
+ (AccNo, GI, hash, ChrAccVer, ChrStart, ChrStop, orientation, url, created)
FROM GBInfo
WHERE created >= %s;
""", created_since
diff --git a/mutalyzer/sync.py b/mutalyzer/sync.py
index b99cf116..1f4a8231 100644
--- a/mutalyzer/sync.py
+++ b/mutalyzer/sync.py
@@ -3,9 +3,14 @@ Module for synchronizing the database with other Mutalyzer instances.
"""
+from mutalyzer.util import monkey_patch_suds; monkey_patch_suds()
+from suds.client import Client
from datetime import datetime, timedelta
+DEFAULT_CREATED_SINCE_DAYS = 7
+
+
class CacheSync(object):
"""
Todo.
@@ -22,5 +27,54 @@ class CacheSync(object):
Todo.
"""
if not created_since:
- created_since = datetime.today() - timedelta(days=7)
+ created_since = datetime.today() - \
+ timedelta(days=DEFAULT_CREATED_SINCE_DAYS)
return self._database.getGB(created_since)
+
+ def remote_cache(self, remote_wsdl, created_since=None):
+ """
+ Todo.
+ """
+ if not created_since:
+ created_since = datetime.today() - \
+ timedelta(days=DEFAULT_CREATED_SINCE_DAYS)
+ client = Client(remote_wsdl, cache=None)
+ cache = client.service.getCache(created_since)
+
+ def cache_entry_from_soap(entry):
+ """
+ Create a nice dictionary out of the CacheEntry object.
+ """
+ entry_dict = {'name': entry.name,
+ 'hash': entry.hash,
+ 'created': entry.created}
+ for attribute in ('gi', 'chromosomeName', 'chromosomeStart'
+ 'chromosomeStop', 'chromosomeOrientation',
+ 'url'):
+ entry_dict[attribute] = entry[attribute] \
+ if attribute in entry else None
+ return entry_dict
+
+ return map(cache_entry_from_soap, cache.CacheEntry)
+
+ def sync_with_remote(self, remote_wsdl, created_since=None):
+ """
+ Todo.
+ """
+ remote_cache = self.remote_cache(remote_wsdl, created_since)
+
+ for entry in remote_cache:
+ if self._database.getHash(entry['name']):
+ continue
+ #self._database.insertGB(entry['name'],
+ # entry['gi'],
+ # entry['hash'],
+ # entry['chromosomeName'],
+ # entry['chromosomeStart'],
+ # entry['chromosomeStop'],
+ # entry['chromosomeOrientation'],
+ # entry['url'])
+ print 'inserting %s' % entry['name']
+ print entry
+ if not entry['chromosomeName'] and not entry['url']:
+ print '(downloading file from remote cache...)'
diff --git a/mutalyzer/util.py b/mutalyzer/util.py
index 892bde65..8aecc08c 100644
--- a/mutalyzer/util.py
+++ b/mutalyzer/util.py
@@ -739,3 +739,30 @@ def slow(f):
f(*args, **kwargs)
return slow_f
#slow
+
+
+def monkey_patch_suds():
+ """
+ Apply our monkey-patch for the suds package.
+
+ For some weird reason the location http://www.w3.org/2001/xml.xsd is used
+ for the XML namespace, but the W3C seems to respond too slow on that url.
+ We therefore use http://www.w3.org/2009/01/xml.xsd which fixes this.
+
+ Call this function before importing anything from the suds package.
+ """
+ from suds.xsd.sxbasic import Import
+ _import_open = Import.open
+
+ # Only apply the patch once.
+ if getattr(Import, 'MUTALYZER_MONKEY_PATCHED', False):
+ return
+
+ def _import_open_patched(self, *args, **kwargs):
+ if self.location == 'http://www.w3.org/2001/xml.xsd':
+ self.location = 'http://www.w3.org/2009/01/xml.xsd'
+ return _import_open(self, *args, **kwargs)
+
+ Import.open = _import_open_patched
+ Import.MUTALYZER_MONKEY_PATCHED = True
+#monkey_patch_suds
diff --git a/mutalyzer/webservice.py b/mutalyzer/webservice.py
index 720ba445..f6401e1d 100644
--- a/mutalyzer/webservice.py
+++ b/mutalyzer/webservice.py
@@ -897,23 +897,23 @@ class MutalyzerService(DefinitionBase):
cache = sync.local_cache(created_since)
- def soap_cache_entry(entry):
+ def cache_entry_to_soap(entry):
e = CacheEntry()
- e.name = entry[0]
- e.gi = entry[1]
- e.hash = entry[2]
- e.chromosomeName = entry[3]
- e.chromosomeStart = entry[4]
- e.chromosomeStop = entry[5]
- e.chromosomeOrientation = entry[6]
- e.url = entry[7]
- e.created = entry[8]
+ (e.name,
+ e.gi,
+ e.hash,
+ e.chromosomeName,
+ e.chromosomeStart,
+ e.chromosomeStop,
+ e.chromosomeOrientation,
+ e.url,
+ e.created) = entry
return e
output.addMessage(__file__, -1, 'INFO',
'Finished processing getCache')
- return map(soap_cache_entry, cache)
+ return map(cache_entry_to_soap, cache)
#getCache
#MutalyzerService
diff --git a/tests/test_webservice.py b/tests/test_webservice.py
index 16019a5b..dd0dc3e6 100644
--- a/tests/test_webservice.py
+++ b/tests/test_webservice.py
@@ -3,21 +3,14 @@ Tests for the SOAP interface to Mutalyzer.
"""
-# Monkey patch suds, because for some weird reason the location
-# http://www.w3.org/2001/xml.xsd is used for the XML namespace, but the W3C
-# seems to respond too slow on that url. We use therefore use
-# http://www.w3.org/2009/01/xml.xsd which fixes this.
-from suds.xsd.sxbasic import Import
-_import_open = Import.open
-def _import_open_patched(self, *args, **kwargs):
- if self.location == 'http://www.w3.org/2001/xml.xsd':
- self.location = 'http://www.w3.org/2009/01/xml.xsd'
- return _import_open(self, *args, **kwargs)
-Import.open = _import_open_patched
-
+from mutalyzer.util import monkey_patch_suds; monkey_patch_suds()
import os
+from datetime import datetime, timedelta
import mutalyzer
+from mutalyzer.config import Config
+from mutalyzer.sync import CacheSync
+from mutalyzer import Db
import logging; logging.raiseExceptions = 0
import urllib2
from suds.client import Client
@@ -169,3 +162,17 @@ class TestWebservice():
r = self.client.service.info()
assert_equal(type(r.versionParts.string), list)
assert_equal(r.version, mutalyzer.__version__)
+
+ def test_getcache(self):
+ """
+ Running the getCache method should give us the expected cache entries.
+ """
+ created_since = datetime.today() - timedelta(days=60)
+
+ config = Config()
+ database = Db.Cache(config.Db)
+ sync = CacheSync(config.Sync, database)
+ cache = sync.local_cache(created_since)
+
+ r = self.client.service.getCache(created_since)
+ assert_equal(len(r.CacheEntry), len(cache))
--
GitLab