Synchronous fetch of a single URL via Jina Reader with retry/backoff and postprocessing.
(url: str, config: JinaReaderConfig)
| 51 | |
| 52 | |
| 53 | def fetch_single_text(url: str, config: JinaReaderConfig) -> str: |
| 54 | """ |
| 55 | Synchronous fetch of a single URL via Jina Reader with retry/backoff and postprocessing. |
| 56 | """ |
| 57 | request_url = _build_reader_url(url, config.base_endpoint) |
| 58 | attempt = 0 |
| 59 | while True: |
| 60 | attempt += 1 |
| 61 | try: |
| 62 | req = Request(request_url, headers=config.headers) |
| 63 | with urlopen(req, timeout=config.timeout) as resp: |
| 64 | data = resp.read() |
| 65 | return _postprocess_text( |
| 66 | data.decode('utf-8', errors='replace')) |
| 67 | except HTTPError as e: |
| 68 | # Retry on 429 and 5xx, otherwise fail fast |
| 69 | status = getattr(e, 'code', None) |
| 70 | if status in (429, 500, 502, 503, |
| 71 | 504) and attempt <= config.retries: |
| 72 | sleep_s = min(config.backoff_max, |
| 73 | config.backoff_base * (2**(attempt - 1))) |
| 74 | sleep_s *= random.uniform(0.7, 1.4) |
| 75 | time.sleep(sleep_s) |
| 76 | continue |
| 77 | return '' |
| 78 | except URLError: |
| 79 | if attempt <= config.retries: |
| 80 | sleep_s = min(config.backoff_max, |
| 81 | config.backoff_base * (2**(attempt - 1))) |
| 82 | sleep_s *= random.uniform(0.7, 1.4) |
| 83 | time.sleep(sleep_s) |
| 84 | continue |
| 85 | return '' |
| 86 | except Exception: |
| 87 | # Unknown error; do not loop excessively |
| 88 | if attempt <= config.retries: |
| 89 | sleep_s = min(config.backoff_max, |
| 90 | config.backoff_base * (2**(attempt - 1))) |
| 91 | sleep_s *= random.uniform(0.7, 1.4) |
| 92 | time.sleep(sleep_s) |
| 93 | continue |
| 94 | return '' |
| 95 | |
| 96 | |
| 97 | async def fetch_texts_via_jina( |
no test coverage detected