(self)
| 101 | return (request, handle) |
| 102 | |
| 103 | def fetch(self): |
| 104 | request, handle = self.open() |
| 105 | self._addHeaders(request) |
| 106 | if handle: |
| 107 | try: |
| 108 | content = unicode(handle.open(request).read(), "utf-8", |
| 109 | errors="replace") |
| 110 | soup = BeautifulSoup(content) |
| 111 | tags = soup('a') |
| 112 | except urllib2.HTTPError, error: |
| 113 | if error.code == 404: |
| 114 | print >> sys.stderr, "ERROR: %s -> %s" % (error, error.url) |
| 115 | else: |
| 116 | print >> sys.stderr, "ERROR: %s" % error |
| 117 | tags = [] |
| 118 | except urllib2.URLError, error: |
| 119 | print >> sys.stderr, "ERROR: %s" % error |
| 120 | tags = [] |
| 121 | for tag in tags: |
| 122 | href = tag.get("href") |
| 123 | if href is not None: |
| 124 | url = urlparse.urljoin(self.url, escape(href)) |
| 125 | if url not in self: |
| 126 | self.urls.append(url) |
| 127 | |
| 128 | def getLinks(url): |
| 129 | page = Fetcher(url) |
no test coverage detected