Method __init__

pattern/web/__init__.py:3107–3125

A crawler can be used to browse the web in an automated manner. It visits the list of starting URLs, parses links from their content, visits those, etc. - Links can be prioritized by overriding Crawler.priority(). - Links can be ignored by overriding Crawler.foll

(self, links=[], domains=[], delay=20.0, parser=HTMLLinkParser().parse, sort=FIFO)

class Crawler(object):

    def __init__(self, links=[], domains=[], delay=20.0, parser=HTMLLinkParser().parse, sort=FIFO):
        """ A crawler can be used to browse the web in an automated manner.
            It visits the list of starting URLs, parses links from their content, visits those, etc.
            - Links can be prioritized by overriding Crawler.priority().
            - Links can be ignored by overriding Crawler.follow().
            - Each visited link is passed to Crawler.visit(), which can be overridden.
        """
        self.parse    = parser
        self.delay    = delay   # Delay between visits to the same (sub)domain.
        self.domains  = domains # Domains the crawler is allowed to visit.
        self.history  = {}      # Domain name => time last visited.
        self.visited  = {}      # URLs visited.
        self._queue   = []      # URLs scheduled for a visit: (priority, time, Link).
        self._queued  = {}      # URLs scheduled so far, lookup dictionary.
        self.QUEUE    = 10000   # Increase or decrease according to available memory.
        self.sort     = sort
        # Queue given links in given order:
        for link in (isinstance(links, basestring) and [links] or links):
            self.push(link, priority=1.0, sort=FIFO)

    @property
    def done(self):