changeset 1:f2147b34d6af

Use pymongo instead of pickling
author Atul Varma <varmaa@toolness.com>
date Wed, 23 Dec 2009 02:23:11 -0800
parents 78e4757601ec
children 4d9c47d9e87e
files bzapi.py example.py
diffstat 2 files changed, 61 insertions(+), 57 deletions(-) [+]
line wrap: on
line diff
--- a/bzapi.py	Tue Dec 22 21:40:25 2009 -0800
+++ b/bzapi.py	Wed Dec 23 02:23:11 2009 -0800
@@ -2,38 +2,69 @@
 import urllib
 from datetime import datetime
 
+import pymongo
 import simplejson as json
 
+def datetime_to_iso(dt):
+    return "%sZ" % (dt.replace(microsecond=0).isoformat('T'))
+
 def datetime_from_iso(timestamp):
     return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
 
+def sanitize(obj):
+    if type(obj) == dict:
+        bad_names = [name for name in obj
+                     if "." in name]
+        for name in bad_names:
+            new_name = name.replace('.', '_DOT_')
+            obj[new_name] = obj[name]
+            del obj[name]
+        for name in obj:
+            sanitize(obj[name])
+    elif type(obj) == list:
+        for item in obj:
+            sanitize(item)
+
 class CachedSearch(object):
-    def __init__(self, api, **kwargs):
+    def __init__(self, api, collection, **kwargs):
         self.options = kwargs
-        self.last_update = None
-        self.bugs = {}
-        self.update(api)
+
+        self.bugs = collection
+        self.api = api
+        self._update_last_update()
 
-    def update(self, api):
+    def _update_last_update(self):
+        bugs = self.bugs.find().sort("last_change_time",
+                                     pymongo.DESCENDING).limit(1)
+        if bugs.count() == 0:
+            self.last_update = None
+        else:
+            self.last_update = bugs[0]['last_change_time']
+
+    def update(self):
         params = {}
         params.update(self.options)
         if self.last_update:
             params['changed_after'] = self.last_update
-        bugs = api.get('/bug', **params)['bugs']
-        latest_time = None
+        bugs = self.api.get('/bug', **params)['bugs']
         for bug in bugs:
-            last_change_time = datetime_from_iso(bug['last_change_time'])
-            if not latest_time or last_change_time > latest_time:
-                latest_time = last_change_time
-            self.bugs[bug['id']] = bug
-        self.last_update = last_change_time
+            for name in ['last_change_time', 'creation_time']:
+                bug[name] = datetime_from_iso(bug[name])
+            bug['_id'] = bug['id']
+            self.bugs.save(bug)
+        self._update_last_update()
 
 class BugzillaApi(object):
-    def __init__(self, base_url, username=None, password=None):
+    def __init__(self, base_url, collection, username=None, password=None):
         self.base_url = base_url
         self.username = username
         self.password = password
-        self.config = self.get('/configuration')
+        config = collection.find_one()
+        if not config:
+            config = self.get('/configuration')
+            sanitize(config)
+            collection.insert(config)
+        self.config = config
 
     def _validate_component(self, product, component=None):
         products = self.config['product']
@@ -50,13 +81,12 @@
     def get(self, url, **kwargs):
         for name, value in kwargs.items():
             if isinstance(value, datetime):
-                kwargs[name] = "%sZ" % (
-                    value.replace(microsecond=0).isoformat('T')
-                    )
+                kwargs[name] = datetime_to_iso(value)
 
+        params = {}
         if self.username and self.password:
-            params = dict(username = self.username,
-                          password = self.password)
+            params.update({'username': self.username,
+                           'password': self.password})
         params.update(kwargs)
 
         if 'product' in params:
--- a/example.py	Tue Dec 22 21:40:25 2009 -0800
+++ b/example.py	Wed Dec 23 02:23:11 2009 -0800
@@ -1,57 +1,31 @@
 import os
 from datetime import datetime, timedelta
-import cPickle as pickle
 
+import pymongo
 import bzapi
 
-class Picklable(object):
-    def __init__(self, filename, constructor, **kwargs):
-        self.__filename = filename
-
-        if not os.path.exists(filename):
-            self.__obj = constructor(**kwargs)
-            self.save()
-        else:
-            obj_file = open(filename, 'r')
-            self.__obj = pickle.load(obj_file)
-            obj_file.close()
-
-    def save(self):
-        obj_file = open(self.__filename, 'w')
-        pickle.dump(self.__obj, obj_file)
-        obj_file.close()
+connection = pymongo.Connection('localhost', 27017)
+db = connection.bzapi_example_db
 
-    def __getattr__(self, attr):
-        if not attr.startswith('_Picklable_'):
-            return getattr(self.__obj, attr)
-        else:
-            return self.__dict__[attr]
-
-    def __setattr__(self, attr, value):
-        if not attr.startswith('_Picklable_'):
-            setattr(self.__obj, attr, value)
-        else:
-            self.__dict__[attr] = value
-
-api = Picklable(
-    'example.api.pickle',
-    bzapi.BugzillaApi,
-    base_url = 'https://api-dev.bugzilla.mozilla.org/latest'
+api = bzapi.BugzillaApi(
+    base_url = 'https://api-dev.bugzilla.mozilla.org/latest',
+    collection = db.api
     )
 
-search = Picklable(
-    'example.cached-search.pickle',
-    bzapi.CachedSearch,
+search = bzapi.CachedSearch(
     api = api,
+    collection = db.bugs,
     product='Mozilla Labs',
     component='Jetpack'
     )
 
-#search.update(api)
+#search.update()
 
 #print len([bug for bug in search.bugs.itervalues()])
 
-#print search.bugs.values()[12]['summary']
+print search.bugs.find_one({'id': '494651'})
+
+#print search.bugs.values()[12]
 
 #print api.get('/bug/510339/history')