Real-use fetch: identify honestly, conditional GET, backoff. (network — live only)
(url: str, ua: str = "QodexBot/1.0 (+contact)", etag: str | None = None,
retries: int = 4, timeout: float = 20.0)
| 35 | return out |
| 36 | |
| 37 | def polite_get(url: str, ua: str = "QodexBot/1.0 (+contact)", etag: str | None = None, |
| 38 | retries: int = 4, timeout: float = 20.0): |
| 39 | """Real-use fetch: identify honestly, conditional GET, backoff. (network — live only)""" |
| 40 | headers = {"User-Agent": ua} |
| 41 | if etag: |
| 42 | headers["If-None-Match"] = etag |
| 43 | delays = backoff_delays(retries) |
| 44 | last = None |
| 45 | for attempt in range(retries + 1): |
| 46 | try: |
| 47 | req = urllib.request.Request(url, headers=headers) |
| 48 | with urllib.request.urlopen(req, timeout=timeout) as r: |
| 49 | return {"status": r.status, "etag": r.headers.get("ETag"), |
| 50 | "body": r.read().decode("utf-8", "replace")} |
| 51 | except urllib.error.HTTPError as e: |
| 52 | if e.code == 304: # not modified — nothing new, that's a success for us |
| 53 | return {"status": 304, "etag": etag, "body": None} |
| 54 | if e.code in (429, 500, 502, 503, 504) and attempt < retries: |
| 55 | ra = e.headers.get("Retry-After") |
| 56 | time.sleep(float(ra) if (ra and ra.isdigit()) else delays[attempt]); last = e; continue |
| 57 | raise |
| 58 | except urllib.error.URLError as e: |
| 59 | if attempt < retries: |
| 60 | time.sleep(delays[attempt]); last = e; continue |
| 61 | raise |
| 62 | raise last # exhausted |
| 63 | |
| 64 | |
| 65 | # --- 3. explicit schema validation: never write a malformed record to the store |
nothing calls this directly
no test coverage detected