| 80 | print format_exc() |
| 81 | |
| 82 | class Fetcher(object): |
| 83 | |
| 84 | def __init__(self, url): |
| 85 | self.url = url |
| 86 | self.urls = [] |
| 87 | |
| 88 | def __getitem__(self, x): |
| 89 | return self.urls[x] |
| 90 | |
| 91 | def _addHeaders(self, request): |
| 92 | request.add_header("User-Agent", AGENT) |
| 93 | |
| 94 | def open(self): |
| 95 | url = self.url |
| 96 | try: |
| 97 | request = urllib2.Request(url) |
| 98 | handle = urllib2.build_opener() |
| 99 | except IOError: |
| 100 | return None |
| 101 | return (request, handle) |
| 102 | |
| 103 | def fetch(self): |
| 104 | request, handle = self.open() |
| 105 | self._addHeaders(request) |
| 106 | if handle: |
| 107 | try: |
| 108 | content = unicode(handle.open(request).read(), "utf-8", |
| 109 | errors="replace") |
| 110 | soup = BeautifulSoup(content) |
| 111 | tags = soup('a') |
| 112 | except urllib2.HTTPError, error: |
| 113 | if error.code == 404: |
| 114 | print >> sys.stderr, "ERROR: %s -> %s" % (error, error.url) |
| 115 | else: |
| 116 | print >> sys.stderr, "ERROR: %s" % error |
| 117 | tags = [] |
| 118 | except urllib2.URLError, error: |
| 119 | print >> sys.stderr, "ERROR: %s" % error |
| 120 | tags = [] |
| 121 | for tag in tags: |
| 122 | href = tag.get("href") |
| 123 | if href is not None: |
| 124 | url = urlparse.urljoin(self.url, escape(href)) |
| 125 | if url not in self: |
| 126 | self.urls.append(url) |
| 127 | |
| 128 | def getLinks(url): |
| 129 | page = Fetcher(url) |