| 27 | |
| 28 | |
| 29 | async def receiveXHR(page, requests): |
| 30 | responses = [] |
| 31 | retries = 0 |
| 32 | max_retries = 5 |
| 33 | # Wait at least 2 seconds after last XHR request for more |
| 34 | while True: |
| 35 | if last_xhr_request is None or retries > max_retries: |
| 36 | break |
| 37 | if time.time() - last_xhr_request <= 2: |
| 38 | retries = retries + 1 |
| 39 | time.sleep(2) |
| 40 | continue |
| 41 | else: |
| 42 | break |
| 43 | await page |
| 44 | # Loop through gathered requests and get response body |
| 45 | for request in requests: |
| 46 | try: |
| 47 | res = await page.send(mycdp.network.get_response_body(request[1])) |
| 48 | if res is None: |
| 49 | continue |
| 50 | responses.append({ |
| 51 | "url": request[0], |
| 52 | "body": res[0], |
| 53 | "is_base64": res[1], |
| 54 | }) |
| 55 | except Exception as e: |
| 56 | print("Error getting response:", e) |
| 57 | return responses |
| 58 | |
| 59 | |
| 60 | async def crawl(): |