Mercurial > daily-edition

annotate whoisi_cache.py @ 23:8b501dfe7d85

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
changed names of cmd line options to make a bit more sense
author Atul Varma <varmaa@toolness.com>
date Sun, 03 Jan 2010 05:01:38 -0800
parents 428d45a1ef44
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
1 import logging
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
2 import urllib
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
3 import urllib2
11
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
4 from copy import deepcopy
7
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
5
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
6 try:
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
7 import json
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
8 except ImportError:
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
9 import simplejson as json
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
10
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
11 MAX_PEOPLE_REQ_SIZE = 100
7
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
12 DEFAULT_URL = "http://whoisi.com/"
11
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
13 EMPTY_PERSON = {'aliases': [],
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
14 'sites': {},
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
15 'name': ''}
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
16
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
17 def split_seq(seq, size):
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
18 """
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
19 Split up the given sequence into pieces of the given size.
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
20
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
21 Taken from http://code.activestate.com/recipes/425044/.
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
22 """
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
23
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
24 return [seq[i:i+size] for i in range(0, len(seq), size)]
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
25
7
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
26 class WhoisiServer(object):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
27 def __init__(self, url=DEFAULT_URL, urlopen=urllib2.urlopen):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
28 self._urlopen = urlopen
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
29 self.url = url
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
30
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
31 def _call_api_method(self, name, query_args):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
32 full_url = "%sapi/%s?%s" % (self.url, name,
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
33 urllib.urlencode(query_args))
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
34 logging.debug('retrieving %s' % full_url)
8
7b6bec689805 forgot to use a file-like object as urlopen's return value
Atul Varma <varmaa@toolness.com>
parents: 7
diff changeset
35 return json.loads(self._urlopen(full_url).read())
7
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
36
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
37 def get_person(self, app, person):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
38 return self._call_api_method('getPerson',
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
39 [('app', app),
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
40 ('person', str(person))])
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
41
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
42 def get_people(self, app, first, last):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
43 return self._call_api_method('getPeople',
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
44 [('app', app),
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
45 ('first', str(first)),
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
46 ('last', str(last))])
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
47
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
48 def get_max_person_id(self, app):
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
49 return self._call_api_method('getMaxPersonID',
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
50 [('app', app)])
98f1da3b585c added WhoisiServer class and tests
Atul Varma <varmaa@toolness.com>
parents: 6
diff changeset
51
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
52 class WhoisiCache(object):
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
53 APP_NAME = "whoisi-cache"
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
54
3
8c35fbdf5f43 added to tests
Atul Varma <varmaa@toolness.com>
parents: 2
diff changeset
55 def __init__(self, server, storage, batch_size=MAX_PEOPLE_REQ_SIZE):
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
56 self.server = server
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
57 self.batch_size = batch_size
3
8c35fbdf5f43 added to tests
Atul Varma <varmaa@toolness.com>
parents: 2
diff changeset
58 self.people = storage
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
59
5
47990d4e6df7 added cache.refresh_people()
Atul Varma <varmaa@toolness.com>
parents: 4
diff changeset
60 def refresh_people(self, people_ids):
47990d4e6df7 added cache.refresh_people()
Atul Varma <varmaa@toolness.com>
parents: 4
diff changeset
61 for person_id in people_ids:
6
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
62 result = self.server.get_person(self.APP_NAME, person_id + 1)
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
63 self.people[person_id] = result['person']
5
47990d4e6df7 added cache.refresh_people()
Atul Varma <varmaa@toolness.com>
parents: 4
diff changeset
64
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
65 def update(self):
10
b6259e713533 fixed another bug
Atul Varma <varmaa@toolness.com>
parents: 8
diff changeset
66 pid = self.server.get_max_person_id(app=self.APP_NAME)['person_id']
2
f9ee6f8f7021 added whoisi_cache.py
Atul Varma <varmaa@toolness.com>
parents:
diff changeset
67 interval = range(len(self.people) + 1, pid + 1)
4
78a2da337f9f added to tests
Atul Varma <varmaa@toolness.com>
parents: 3
diff changeset
68 subintervals = split_seq(interval, self.batch_size)
78a2da337f9f added to tests
Atul Varma <varmaa@toolness.com>
parents: 3
diff changeset
69 for subinterval in subintervals:
6
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
70 first = subinterval[0]
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
71 last = subinterval[-1]
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
72 result = self.server.get_people(app=self.APP_NAME,
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
73 first=first,
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
74 last=last)
5a9e578c80fa changed around protocol a bit
Atul Varma <varmaa@toolness.com>
parents: 5
diff changeset
75 for person_id in range(first, last + 1):
11
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
76 if str(person_id) in result['people']:
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
77 person = result['people'][str(person_id)]
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
78 else:
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
79 person = deepcopy(EMPTY_PERSON)
428d45a1ef44 dealt with case in which whoisi doesn't include deleted folks
Atul Varma <varmaa@toolness.com>
parents: 10
diff changeset
80 self.people.append(person)