Mercurial > bzapi
changeset 1:f2147b34d6af
Use pymongo instead of pickling
author | Atul Varma <varmaa@toolness.com> |
---|---|
date | Wed, 23 Dec 2009 02:23:11 -0800 |
parents | 78e4757601ec |
children | 4d9c47d9e87e |
files | bzapi.py example.py |
diffstat | 2 files changed, 61 insertions(+), 57 deletions(-) [+] |
line wrap: on
line diff
--- a/bzapi.py Tue Dec 22 21:40:25 2009 -0800 +++ b/bzapi.py Wed Dec 23 02:23:11 2009 -0800 @@ -2,38 +2,69 @@ import urllib from datetime import datetime +import pymongo import simplejson as json +def datetime_to_iso(dt): + return "%sZ" % (dt.replace(microsecond=0).isoformat('T')) + def datetime_from_iso(timestamp): return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ') +def sanitize(obj): + if type(obj) == dict: + bad_names = [name for name in obj + if "." in name] + for name in bad_names: + new_name = name.replace('.', '_DOT_') + obj[new_name] = obj[name] + del obj[name] + for name in obj: + sanitize(obj[name]) + elif type(obj) == list: + for item in obj: + sanitize(item) + class CachedSearch(object): - def __init__(self, api, **kwargs): + def __init__(self, api, collection, **kwargs): self.options = kwargs - self.last_update = None - self.bugs = {} - self.update(api) + + self.bugs = collection + self.api = api + self._update_last_update() - def update(self, api): + def _update_last_update(self): + bugs = self.bugs.find().sort("last_change_time", + pymongo.DESCENDING).limit(1) + if bugs.count() == 0: + self.last_update = None + else: + self.last_update = bugs[0]['last_change_time'] + + def update(self): params = {} params.update(self.options) if self.last_update: params['changed_after'] = self.last_update - bugs = api.get('/bug', **params)['bugs'] - latest_time = None + bugs = self.api.get('/bug', **params)['bugs'] for bug in bugs: - last_change_time = datetime_from_iso(bug['last_change_time']) - if not latest_time or last_change_time > latest_time: - latest_time = last_change_time - self.bugs[bug['id']] = bug - self.last_update = last_change_time + for name in ['last_change_time', 'creation_time']: + bug[name] = datetime_from_iso(bug[name]) + bug['_id'] = bug['id'] + self.bugs.save(bug) + self._update_last_update() class BugzillaApi(object): - def __init__(self, base_url, username=None, password=None): + def __init__(self, base_url, collection, username=None, password=None): self.base_url = base_url self.username = username self.password = password - self.config = self.get('/configuration') + config = collection.find_one() + if not config: + config = self.get('/configuration') + sanitize(config) + collection.insert(config) + self.config = config def _validate_component(self, product, component=None): products = self.config['product'] @@ -50,13 +81,12 @@ def get(self, url, **kwargs): for name, value in kwargs.items(): if isinstance(value, datetime): - kwargs[name] = "%sZ" % ( - value.replace(microsecond=0).isoformat('T') - ) + kwargs[name] = datetime_to_iso(value) + params = {} if self.username and self.password: - params = dict(username = self.username, - password = self.password) + params.update({'username': self.username, + 'password': self.password}) params.update(kwargs) if 'product' in params:
--- a/example.py Tue Dec 22 21:40:25 2009 -0800 +++ b/example.py Wed Dec 23 02:23:11 2009 -0800 @@ -1,57 +1,31 @@ import os from datetime import datetime, timedelta -import cPickle as pickle +import pymongo import bzapi -class Picklable(object): - def __init__(self, filename, constructor, **kwargs): - self.__filename = filename - - if not os.path.exists(filename): - self.__obj = constructor(**kwargs) - self.save() - else: - obj_file = open(filename, 'r') - self.__obj = pickle.load(obj_file) - obj_file.close() - - def save(self): - obj_file = open(self.__filename, 'w') - pickle.dump(self.__obj, obj_file) - obj_file.close() +connection = pymongo.Connection('localhost', 27017) +db = connection.bzapi_example_db - def __getattr__(self, attr): - if not attr.startswith('_Picklable_'): - return getattr(self.__obj, attr) - else: - return self.__dict__[attr] - - def __setattr__(self, attr, value): - if not attr.startswith('_Picklable_'): - setattr(self.__obj, attr, value) - else: - self.__dict__[attr] = value - -api = Picklable( - 'example.api.pickle', - bzapi.BugzillaApi, - base_url = 'https://api-dev.bugzilla.mozilla.org/latest' +api = bzapi.BugzillaApi( + base_url = 'https://api-dev.bugzilla.mozilla.org/latest', + collection = db.api ) -search = Picklable( - 'example.cached-search.pickle', - bzapi.CachedSearch, +search = bzapi.CachedSearch( api = api, + collection = db.bugs, product='Mozilla Labs', component='Jetpack' ) -#search.update(api) +#search.update() #print len([bug for bug in search.bugs.itervalues()]) -#print search.bugs.values()[12]['summary'] +print search.bugs.find_one({'id': '494651'}) + +#print search.bugs.values()[12] #print api.get('/bug/510339/history')