(offset: int, attempt: int = 1)
| 21 | PAGE = 100 |
| 22 | |
| 23 | |
| 24 | def fetch(offset: int, attempt: int = 1): |
| 25 | url = (f'{BASE}?dataset={DATASET}&config=all&split=train' |
| 26 | f'&offset={offset}&length={PAGE}') |
| 27 | try: |
| 28 | req = urllib.request.Request(url, headers={'User-Agent': 'curl/8.0'}) |
| 29 | with urllib.request.urlopen(req, timeout=60) as resp: |
| 30 | return json.loads(resp.read().decode('utf-8')) |
| 31 | except Exception as e: |
| 32 | if attempt < 3: |
| 33 | print(f' offset={offset} attempt {attempt} failed: {e}, retrying...') |
| 34 | time.sleep(5 * attempt) |
| 35 | return fetch(offset, attempt + 1) |
| 36 | print(f' offset={offset} FAILED: {e}, skipping') |
| 37 | return None |
| 38 | |
| 39 |
no test coverage detected