MCPcopy Create free account
hub / github.com/CScorza/IntelOSINT / _safe_request

Method _safe_request

IntelOSINT.py:572–597  ·  view source on GitHub ↗
(self, session, url, *, cookies=None, timeout=20, max_retries=5, stream=False, headers=None)

Source from the content-addressed store, hash-verified

570 time.sleep(delay)
571
572 def _safe_request(self, session, url, *, cookies=None, timeout=20, max_retries=5, stream=False, headers=None):
573 # GET resiliente con backoff esplicito su 429 e retry automatici.
574 req_headers = headers or {}
575 if req_headers:
576 session.headers.update(req_headers)
577 last_error = None
578 for attempt in range(max_retries):
579 try:
580 response = session.get(url, cookies=cookies, timeout=timeout, allow_redirects=True, stream=stream)
581 if response.status_code == 429:
582 wait = (2 ** attempt) + random.uniform(0.3, 1.4)
583 time.sleep(wait)
584 continue
585 if response.status_code in (403, 404):
586 # Evita loop inutili su codici non temporanei.
587 return response
588 if response.status_code >= 500:
589 time.sleep((2 ** attempt) + random.uniform(0.5, 1.2))
590 continue
591 return response
592 except requests.exceptions.RequestException as e:
593 last_error = e
594 time.sleep((2 ** attempt) + random.uniform(0.5, 1.5))
595 if last_error is not None:
596 raise last_error
597 raise RuntimeError(f"HTTP request fallita: {url}")
598
599 def start_task(self, target, platform, options):
600 task_id = str(uuid.uuid4())

Callers 2

_scrape_instagramMethod · 0.95
_scrape_tiktokMethod · 0.95

Calls

no outgoing calls

Tested by

no test coverage detected