crawl main method
(self)
| 32 | yield proxy |
| 33 | |
| 34 | def crawl(self): |
| 35 | """ |
| 36 | crawl main method |
| 37 | """ |
| 38 | try: |
| 39 | for url in self.urls: |
| 40 | logger.info(f'fetching {url}') |
| 41 | html = self.fetch(url) |
| 42 | if not html: |
| 43 | continue |
| 44 | time.sleep(.5) |
| 45 | yield from self.process(html, url) |
| 46 | except RetryError: |
| 47 | logger.error( |
| 48 | f'crawler {self} crawled proxy unsuccessfully, ' |
| 49 | 'please check if target url is valid or network issue') |