Subclass of the HTMLParser object. Records the HREF attributes of anchor tags if the scheme is 'http' and the anchor occurs in the 'content' section of the page.
| 10 | HTTP_TIMEOUT = 60.0 # Max. seconds to wait for a response |
| 11 | |
| 12 | class UrlFinder(HTMLParser): |
| 13 | |
| 14 | '''Subclass of the HTMLParser object. Records the HREF attributes |
| 15 | of anchor tags if the scheme is 'http' and the anchor occurs in |
| 16 | the 'content' section of the page.''' |
| 17 | |
| 18 | def __init__(self): |
| 19 | HTMLParser.__init__(self) |
| 20 | self.mirrorLinks = [] |
| 21 | |
| 22 | # True if we're currently in the 'content' section |
| 23 | self.isInMirrors = False |
| 24 | |
| 25 | def handle_comment(self, data): |
| 26 | |
| 27 | # The comments have spaces before and after, but don't count |
| 28 | # on that. |
| 29 | data = data.strip() |
| 30 | |
| 31 | if 'content BEGIN' == data: |
| 32 | self.isInMirrors = True |
| 33 | elif 'content END' == data: |
| 34 | self.isInMirrors = False |
| 35 | |
| 36 | def handle_starttag(self, tag, attrs): |
| 37 | if self.isInMirrors: |
| 38 | attrs = dict(attrs) # Convert from tuple of tuples to dict |
| 39 | if 'a' == tag and 'http' == urllib.splittype(attrs['href'])[0]: |
| 40 | self.mirrorLinks.append(attrs['href']) |
| 41 | |
| 42 | # Record the start time, so we can print a nice message at the end |
| 43 | processStartTime = time.time() |