Returns the next Link queued to visit and removes it from the queue. Links on a recently visited (sub)domain are skipped until Crawler.delay has elapsed.
(self, remove=True)
| 3145 | self._queued[link.url] = True |
| 3146 | |
| 3147 | def pop(self, remove=True): |
| 3148 | """ Returns the next Link queued to visit and removes it from the queue. |
| 3149 | Links on a recently visited (sub)domain are skipped until Crawler.delay has elapsed. |
| 3150 | """ |
| 3151 | now = time.time() |
| 3152 | for i, (priority, dt, link) in enumerate(self._queue): |
| 3153 | if self.delay <= now - self.history.get(base(link.url), 0): |
| 3154 | if remove is True: |
| 3155 | self._queue.pop(i) |
| 3156 | self._queued.pop(link.url, None) |
| 3157 | return link |
| 3158 | |
| 3159 | @property |
| 3160 | def next(self): |