| 32 | AGENT = "%s/%s" % (__name__, __version__) |
| 33 | |
| 34 | class Crawler(object): |
| 35 | |
| 36 | def __init__(self, root, depth, locked=True): |
| 37 | self.root = root |
| 38 | self.depth = depth |
| 39 | self.locked = locked |
| 40 | self.host = urlparse.urlparse(root)[1] |
| 41 | self.urls = [] |
| 42 | self.links = 0 |
| 43 | self.followed = 0 |
| 44 | |
| 45 | def crawl(self): |
| 46 | page = Fetcher(self.root) |
| 47 | page.fetch() |
| 48 | q = Queue() |
| 49 | for url in page.urls: |
| 50 | q.put(url) |
| 51 | followed = [self.root] |
| 52 | |
| 53 | n = 0 |
| 54 | |
| 55 | while True: |
| 56 | try: |
| 57 | url = q.get() |
| 58 | except QueueEmpty: |
| 59 | break |
| 60 | |
| 61 | n += 1 |
| 62 | |
| 63 | if url not in followed: |
| 64 | try: |
| 65 | host = urlparse.urlparse(url)[1] |
| 66 | if self.locked and re.match(".*%s" % self.host, host): |
| 67 | followed.append(url) |
| 68 | self.followed += 1 |
| 69 | page = Fetcher(url) |
| 70 | page.fetch() |
| 71 | for i, url in enumerate(page): |
| 72 | if url not in self.urls: |
| 73 | self.links += 1 |
| 74 | q.put(url) |
| 75 | self.urls.append(url) |
| 76 | if n > self.depth and self.depth > 0: |
| 77 | break |
| 78 | except Exception, e: |
| 79 | print "ERROR: Can't process url '%s' (%s)" % (url, e) |
| 80 | print format_exc() |
| 81 | |
| 82 | class Fetcher(object): |
| 83 | |