Download content with connection reuse and redirect handling. Args: url: URL to download from Returns: tuple containing: bytes: Downloaded content str: Final URL after any redirects
(self, url: str)
| 413 | return url |
| 414 | |
| 415 | def download(self, url: str) -> tuple[bytes, str]: |
| 416 | """ |
| 417 | Download content with connection reuse and redirect handling. |
| 418 | |
| 419 | Args: |
| 420 | url: URL to download from |
| 421 | |
| 422 | Returns: |
| 423 | tuple containing: |
| 424 | bytes: Downloaded content |
| 425 | str: Final URL after any redirects |
| 426 | |
| 427 | Features: |
| 428 | - Connection pooling/reuse |
| 429 | - Redirect caching |
| 430 | - Rate limiting |
| 431 | - Host fallback on failure |
| 432 | - Automatic retries |
| 433 | """ |
| 434 | now = time.time() |
| 435 | wait_time = self.last_request_time + self.min_request_interval - now |
| 436 | if (wait_time > 0): |
| 437 | time.sleep(wait_time) |
| 438 | |
| 439 | try: |
| 440 | # Use cached redirect if available |
| 441 | if url in self.redirect_cache: |
| 442 | logging.debug(f"Using cached redirect for {url}") |
| 443 | final_url = self.redirect_cache[url] |
| 444 | response = self.session.get(final_url, timeout=10) |
| 445 | else: |
| 446 | response = self.session.get(url, allow_redirects=True, timeout=10) |
| 447 | if response.history: # Cache redirects |
| 448 | logging.debug(f"Caching redirect for {url} -> {response.url}") |
| 449 | self.redirect_cache[url] = response.url |
| 450 | |
| 451 | self.last_request_time = time.time() |
| 452 | |
| 453 | if response.status_code == 200: |
| 454 | self.last_host = self.get_base_host(response.url) |
| 455 | |
| 456 | return response.content, response.url |
| 457 | |
| 458 | except Exception as e: |
| 459 | logging.error(f"Download error: {e}") |
| 460 | if self.last_host and not url.startswith(self.last_host): |
| 461 | # Use urljoin to handle path resolution |
| 462 | new_url = urljoin(self.last_host + '/', url.split('://')[-1].split('/', 1)[-1]) |
| 463 | logging.debug(f"Retrying with last host: {new_url}") |
| 464 | return self.download(new_url) |
| 465 | raise |
| 466 | |
| 467 | def fetch_loop(self): |
| 468 | """Main fetch loop for stream data""" |
no test coverage detected