| 980 | print "pattern.web.HTMLLinkParser" |
| 981 | |
| 982 | def test_crawler_crawl(self): |
| 983 | # Assert domain filter. |
| 984 | v = web.Crawler(links=["http://www.clips.ua.ac.be/"], domains=["clips.ua.ac.be"], delay=0.5) |
| 985 | while len(v.visited) < 4: |
| 986 | v.crawl(throttle=0.1, cached=False) |
| 987 | for url in v.visited: |
| 988 | self.assertTrue("clips.ua.ac.be" in url) |
| 989 | self.assertTrue(len(v.history) == 1) |
| 990 | print "pattern.web.Crawler.crawl()" |
| 991 | |
| 992 | def test_crawler_delay(self): |
| 993 | # Assert delay for several crawls to a single domain. |