Makes an HTTP request while automatically: - avoiding rate limits (sleep/retry) - cycling API keys - cancelling after too many failed attempts
(self, *args, **kwargs)
| 1259 | return url, kwargs |
| 1260 | |
| 1261 | async def api_request(self, *args, **kwargs): |
| 1262 | """ |
| 1263 | Makes an HTTP request while automatically: |
| 1264 | - avoiding rate limits (sleep/retry) |
| 1265 | - cycling API keys |
| 1266 | - cancelling after too many failed attempts |
| 1267 | """ |
| 1268 | url = args[0] if args else kwargs.pop("url", "") |
| 1269 | retry_on_http_429 = kwargs.pop("retry_on_http_429", True) |
| 1270 | |
| 1271 | # loop until we have a successful request |
| 1272 | for _ in range(self.api_retries): |
| 1273 | if "headers" not in kwargs: |
| 1274 | kwargs["headers"] = {} |
| 1275 | new_url, kwargs = self.prepare_api_request(url, kwargs) |
| 1276 | kwargs["url"] = new_url |
| 1277 | |
| 1278 | r = await self.helpers.request(**kwargs) |
| 1279 | success = r is not None and self._api_response_is_success(r) |
| 1280 | |
| 1281 | if success: |
| 1282 | self._api_request_failures = 0 |
| 1283 | else: |
| 1284 | status_code = getattr(r, "status_code", 0) |
| 1285 | response_text = getattr(r, "text", "") |
| 1286 | self.trace(f"API response to {url} failed with status code {status_code}: {response_text}") |
| 1287 | self._api_request_failures += 1 |
| 1288 | if self._api_request_failures >= self.api_failure_abort_threshold: |
| 1289 | self.set_error_state( |
| 1290 | f"Setting error state due to {self._api_request_failures:,} failed HTTP requests" |
| 1291 | ) |
| 1292 | else: |
| 1293 | # sleep for a bit if we're being rate limited |
| 1294 | retry_after = self._get_retry_after(r) |
| 1295 | if (retry_after or status_code == 429) and retry_on_http_429: |
| 1296 | sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval |
| 1297 | if retry_after and retry_after > self._429_max_sleep_interval: |
| 1298 | self.verbose( |
| 1299 | f"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead" |
| 1300 | ) |
| 1301 | sleep_interval = self._429_max_sleep_interval |
| 1302 | self.verbose( |
| 1303 | f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})" |
| 1304 | ) |
| 1305 | await asyncio.sleep(sleep_interval) |
| 1306 | elif self._api_keys: |
| 1307 | # if request failed, cycle API keys and try again |
| 1308 | self.cycle_api_key() |
| 1309 | continue |
| 1310 | break |
| 1311 | |
| 1312 | return r |
| 1313 | |
| 1314 | async def api_download(self, url, **kwargs): |
| 1315 | """ |
no test coverage detected