Mercurial > daily-edition
view whoisi_cache.py @ 29:6fbd38dd976a
added more logging stats.
author | Atul Varma <varmaa@toolness.com> |
---|---|
date | Sun, 03 Jan 2010 10:21:53 -0800 |
parents | 428d45a1ef44 |
children |
line wrap: on
line source
import logging import urllib import urllib2 from copy import deepcopy try: import json except ImportError: import simplejson as json MAX_PEOPLE_REQ_SIZE = 100 DEFAULT_URL = "http://whoisi.com/" EMPTY_PERSON = {'aliases': [], 'sites': {}, 'name': ''} def split_seq(seq, size): """ Split up the given sequence into pieces of the given size. Taken from http://code.activestate.com/recipes/425044/. """ return [seq[i:i+size] for i in range(0, len(seq), size)] class WhoisiServer(object): def __init__(self, url=DEFAULT_URL, urlopen=urllib2.urlopen): self._urlopen = urlopen self.url = url def _call_api_method(self, name, query_args): full_url = "%sapi/%s?%s" % (self.url, name, urllib.urlencode(query_args)) logging.debug('retrieving %s' % full_url) return json.loads(self._urlopen(full_url).read()) def get_person(self, app, person): return self._call_api_method('getPerson', [('app', app), ('person', str(person))]) def get_people(self, app, first, last): return self._call_api_method('getPeople', [('app', app), ('first', str(first)), ('last', str(last))]) def get_max_person_id(self, app): return self._call_api_method('getMaxPersonID', [('app', app)]) class WhoisiCache(object): APP_NAME = "whoisi-cache" def __init__(self, server, storage, batch_size=MAX_PEOPLE_REQ_SIZE): self.server = server self.batch_size = batch_size self.people = storage def refresh_people(self, people_ids): for person_id in people_ids: result = self.server.get_person(self.APP_NAME, person_id + 1) self.people[person_id] = result['person'] def update(self): pid = self.server.get_max_person_id(app=self.APP_NAME)['person_id'] interval = range(len(self.people) + 1, pid + 1) subintervals = split_seq(interval, self.batch_size) for subinterval in subintervals: first = subinterval[0] last = subinterval[-1] result = self.server.get_people(app=self.APP_NAME, first=first, last=last) for person_id in range(first, last + 1): if str(person_id) in result['people']: person = result['people'][str(person_id)] else: person = deepcopy(EMPTY_PERSON) self.people.append(person)