| 19 | |
| 20 | |
| 21 | async def fetch_resource(url, request: Request, timeout, proxy=None): |
| 22 | logger.info(f"Begin to download resource from url: {url}") |
| 23 | start_time = time.time() |
| 24 | async with httpx.AsyncClient(proxy=proxy) as client: |
| 25 | async with client.stream("GET", url, timeout=timeout) as response: |
| 26 | response.raise_for_status() |
| 27 | ans_bytes = [] |
| 28 | async for chunk in response.aiter_bytes(chunk_size=1024 * 1024): |
| 29 | if request is not None and await request.is_disconnected(): |
| 30 | await response.aclose() |
| 31 | raise Exception("Request disconnected. User cancelled download.") |
| 32 | ans_bytes.append(chunk) |
| 33 | # 接收的数据不能大于128M |
| 34 | if len(ans_bytes) > 128: |
| 35 | raise Exception(f"url {url} recv data is too big") |
| 36 | |
| 37 | content = b"".join(ans_bytes) |
| 38 | end_time = time.time() |
| 39 | cost_time = end_time - start_time |
| 40 | logger.info(f"Download url {url} resource cost time: {cost_time} seconds") |
| 41 | return content |