| 15 | demo = True # Settings for demo mode |
| 16 | |
| 17 | def get_all_pages(url, headers, params=None): |
| 18 | results = [] |
| 19 | page = 1 |
| 20 | while True: |
| 21 | logging.debug(f"Fetching page {page} for URL: {url} with params: {params}") |
| 22 | req_params = {} if not params else params.copy() |
| 23 | req_params.update({"per_page": 100, "page": page}) |
| 24 | resp = requests.get(url, headers=headers, params=req_params) |
| 25 | resp.raise_for_status() |
| 26 | items = resp.json() |
| 27 | if not items: |
| 28 | logging.debug("No items returned, ending pagination") |
| 29 | break |
| 30 | results.extend(items) |
| 31 | logging.debug(f"Page {page} returned {len(items)} items") |
| 32 | page += 1 |
| 33 | return results |
| 34 | |
| 35 | |
| 36 | def main(): |