| 546 | import zipfile |
| 547 | |
| 548 | class ScrapeTaskManager: |
| 549 | def __init__(self): |
| 550 | self.tasks = {} |
| 551 | |
| 552 | def _scrape_headers(self, platform): |
| 553 | # Header/cookie profile più realistico e riusabile tra richieste HTTP e Playwright. |
| 554 | ua_list = [ |
| 555 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", |
| 556 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", |
| 557 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15" |
| 558 | ] |
| 559 | return { |
| 560 | "User-Agent": random.choice(ua_list), |
| 561 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", |
| 562 | "Accept-Language": "en-US,en;q=0.9,it;q=0.8", |
| 563 | "Referer": f"https://www.{platform}.com/", |
| 564 | "Cache-Control": "no-cache", |
| 565 | } |
| 566 | |
| 567 | def _rand_sleep(self, base=0.7, jitter=0.8): |
| 568 | # Sleep con jitter per ridurre pattern ripetitivi e sembrare più umano. |
| 569 | delay = base + random.random() * jitter |
| 570 | time.sleep(delay) |
| 571 | |
| 572 | def _safe_request(self, session, url, *, cookies=None, timeout=20, max_retries=5, stream=False, headers=None): |
| 573 | # GET resiliente con backoff esplicito su 429 e retry automatici. |
| 574 | req_headers = headers or {} |
| 575 | if req_headers: |
| 576 | session.headers.update(req_headers) |
| 577 | last_error = None |
| 578 | for attempt in range(max_retries): |
| 579 | try: |
| 580 | response = session.get(url, cookies=cookies, timeout=timeout, allow_redirects=True, stream=stream) |
| 581 | if response.status_code == 429: |
| 582 | wait = (2 ** attempt) + random.uniform(0.3, 1.4) |
| 583 | time.sleep(wait) |
| 584 | continue |
| 585 | if response.status_code in (403, 404): |
| 586 | # Evita loop inutili su codici non temporanei. |
| 587 | return response |
| 588 | if response.status_code >= 500: |
| 589 | time.sleep((2 ** attempt) + random.uniform(0.5, 1.2)) |
| 590 | continue |
| 591 | return response |
| 592 | except requests.exceptions.RequestException as e: |
| 593 | last_error = e |
| 594 | time.sleep((2 ** attempt) + random.uniform(0.5, 1.5)) |
| 595 | if last_error is not None: |
| 596 | raise last_error |
| 597 | raise RuntimeError(f"HTTP request fallita: {url}") |
| 598 | |
| 599 | def start_task(self, target, platform, options): |
| 600 | task_id = str(uuid.uuid4()) |
| 601 | self.tasks[task_id] = { |
| 602 | 'target': target, |
| 603 | 'platform': platform, |
| 604 | 'options': options, |
| 605 | 'status': 'running', |