import os
import cPickle
import urllib2
import distutils.dir_util

import generate_index

BASE_URL = "http://ifarchive.flavorplex.com"

if __name__ == "__main__":
    dicts = cPickle.load(open(generate_index.PICKLE_FILENAME, "r"))
    for info in dicts:
        path = info['path']
        if not os.path.exists(path):
            url = "%s/%s" % (BASE_URL, path)
            print "Fetching %s..." % url
            try:
                fileobj = urllib2.urlopen(url)
                contents = fileobj.read()
                distutils.dir_util.mkpath(os.path.dirname(path))
                open(path, "wb").write(contents)
            except urllib2.HTTPError, e:
                if e.code == 403:
                    print "  Forbidden, skipping."
                else:
                    raise