Function crawl

pattern/web/__init__.py:3277–3301

Returns a generator that yields (Link, source)-tuples of visited pages. When the crawler is busy, it yields (None, None). When the crawler is done, it yields None.

(links=[], domains=[], delay=20.0, parser=HTMLLinkParser().parse, sort=FIFO, method=DEPTH, **kwargs)

# Functional approach to crawling.

def crawl(links=[], domains=[], delay=20.0, parser=HTMLLinkParser().parse, sort=FIFO, method=DEPTH, **kwargs):
    """ Returns a generator that yields (Link, source)-tuples of visited pages.
        When the crawler is busy, it yields (None, None).
        When the crawler is done, it yields None.
    """
    # The scenarios below defines "busy":
    # - crawl(delay=10, throttle=0)
    #   The crawler will wait 10 seconds before visiting the same subdomain.
    #   The crawler will not throttle downloads, so the next link is visited instantly.
    #   So sometimes (None, None) is returned while it waits for an available subdomain.
    # - crawl(delay=0, throttle=10)
    #   The crawler will halt 10 seconds after each visit.
    #   The crawler will not delay before visiting the same subdomain.
    #   So usually a result is returned each crawl.next(), but each call takes 10 seconds.
    # - asynchronous(crawl().next)
    #   AsynchronousRequest.value is set to (Link, source) once AsynchronousRequest.done=True.
    #   The program will not halt in the meantime (i.e., the next crawl is threaded).
    crawler = Crawler(links, domains, delay, parser, sort)
    bind(crawler, "visit", \
        lambda crawler, link, source=None: \
            setattr(crawler, "crawled", (link, source))) # Define Crawler.visit() on-the-fly.
    while not crawler.done:
        crawler.crawled = (None, None)
        crawler.crawl(method, **kwargs)
        yield crawler.crawled

#for link, source in crawl("http://www.nodebox.net/", delay=0, throttle=10):
#    print link