(self, url, **kwargs)
| 11 | |
| 12 | @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000) |
| 13 | def fetch(self, url, **kwargs): |
| 14 | try: |
| 15 | headers = Headers(headers=True).generate() |
| 16 | kwargs.setdefault('timeout', GET_TIMEOUT) |
| 17 | kwargs.setdefault('verify', False) |
| 18 | kwargs.setdefault('headers', headers) |
| 19 | response = requests.get(url, **kwargs) |
| 20 | if response.status_code == 200: |
| 21 | response.encoding = 'utf-8' |
| 22 | return response.text |
| 23 | except (requests.ConnectionError, requests.ReadTimeout): |
| 24 | return |
| 25 | |
| 26 | def process(self, html, url): |
| 27 | """ |
no outgoing calls
no test coverage detected