Iteratively gets data by calling fetch_func with a moving offset and a limit. Once fetch_func returns None, the retrieval is completed.
(fetch_func: Callable[[int, int], list], limit: int, offset: int, max_req_limit: int = 40,
unpack: bool = True)
| 68 | |
| 69 | |
| 70 | def api_iterative_data(fetch_func: Callable[[int, int], list], limit: int, offset: int, max_req_limit: int = 40, |
| 71 | unpack: bool = True) -> list: |
| 72 | """ |
| 73 | Iteratively gets data by calling fetch_func with a moving offset and a limit. |
| 74 | Once fetch_func returns None, the retrieval is completed. |
| 75 | """ |
| 76 | if limit is None: |
| 77 | limit = max_req_limit |
| 78 | |
| 79 | end = offset + limit |
| 80 | api_data = [] |
| 81 | for offs in range(offset, end, max_req_limit): |
| 82 | # Mimic actual scratch by only requesting the max amount |
| 83 | data = fetch_func(offs, max_req_limit) |
| 84 | if data is None: |
| 85 | break |
| 86 | |
| 87 | if unpack: |
| 88 | api_data.extend(data) |
| 89 | else: |
| 90 | api_data.append(data) |
| 91 | |
| 92 | if len(data) < max_req_limit: |
| 93 | break |
| 94 | |
| 95 | api_data = api_data[:limit] |
| 96 | return api_data |
| 97 | |
| 98 | |
| 99 | def api_iterative(url: str, *, limit: int, offset: int, max_req_limit: int = 40, add_params: str = "", |
no test coverage detected