Class Crawler

pattern/web/__init__.py:3105–3260

LIFO = "lifo" # Last In, First Out (= FILO).

class Crawler(object):

    def __init__(self, links=[], domains=[], delay=20.0, parser=HTMLLinkParser().parse, sort=FIFO):
        """ A crawler can be used to browse the web in an automated manner.
            It visits the list of starting URLs, parses links from their content, visits those, etc.
            - Links can be prioritized by overriding Crawler.priority().
            - Links can be ignored by overriding Crawler.follow().
            - Each visited link is passed to Crawler.visit(), which can be overridden.
        """
        self.parse    = parser
        self.delay    = delay   # Delay between visits to the same (sub)domain.
        self.domains  = domains # Domains the crawler is allowed to visit.
        self.history  = {}      # Domain name => time last visited.
        self.visited  = {}      # URLs visited.
        self._queue   = []      # URLs scheduled for a visit: (priority, time, Link).
        self._queued  = {}      # URLs scheduled so far, lookup dictionary.
        self.QUEUE    = 10000   # Increase or decrease according to available memory.
        self.sort     = sort
        # Queue given links in given order:
        for link in (isinstance(links, basestring) and [links] or links):
            self.push(link, priority=1.0, sort=FIFO)

    @property
    def done(self):
        """ Yields True if no further links are scheduled to visit.
        """
        return len(self._queue) == 0

    def push(self, link, priority=1.0, sort=FILO):
        """ Pushes the given link to the queue.
            Position in the queue is determined by priority.
            Equal ranks are sorted FIFO or FILO.
            With priority=1.0 and FILO, the link is inserted to the queue.
            With priority=0.0 and FIFO, the link is appended to the queue.
        """
        if not isinstance(link, Link):
            link = Link(url=link)
        dt = time.time()
        dt = sort == FIFO and dt or 1 / dt
        bisect.insort(self._queue, (1 - priority, dt, link))
        self._queued[link.url] = True

    def pop(self, remove=True):
        """ Returns the next Link queued to visit and removes it from the queue.
            Links on a recently visited (sub)domain are skipped until Crawler.delay has elapsed.
        """
        now = time.time()
        for i, (priority, dt, link) in enumerate(self._queue):
            if self.delay <= now - self.history.get(base(link.url), 0):
                if remove is True:
                    self._queue.pop(i)
                    self._queued.pop(link.url, None)
                return link

    @property
    def next(self):
        """ Returns the next Link queued to visit (without removing it).
        """