(self)
| 990 | print "pattern.web.Crawler.crawl()" |
| 991 | |
| 992 | def test_crawler_delay(self): |
| 993 | # Assert delay for several crawls to a single domain. |
| 994 | v = web.Crawler(links=["http://www.clips.ua.ac.be/"], domains=["clips.ua.ac.be"], delay=1.0) |
| 995 | v.crawl() |
| 996 | t = time.time() |
| 997 | while not v.crawl(throttle=0.1, cached=False): |
| 998 | pass |
| 999 | t = time.time() - t |
| 1000 | self.assertTrue(t > 1.0) |
| 1001 | print "pattern.web.Crawler.delay" |
| 1002 | |
| 1003 | def test_crawler_breadth(self): |
| 1004 | # Assert BREADTH cross-domain preference. |