Async HTTP client with rate limiting and common configurations.
| 43 | self.tokens -= 1 |
| 44 | |
| 45 | |
| 46 | class HTTPClient: |
| 47 | """Async HTTP client with rate limiting and common configurations.""" |
| 48 | |
| 49 | def __init__( |
| 50 | self, |
| 51 | base_url: str | None = None, |
| 52 | timeout: int | None = None, |
| 53 | rate_limit: float | None = None, |
| 54 | headers: dict[str, str] | None = None, |
| 55 | ): |
| 56 | settings = get_settings() |
| 57 | self.timeout = timeout or settings.request_timeout |
| 58 | self.rate_limiter = RateLimiter(rate_limit or settings.requests_per_second) |
| 59 | |
| 60 | default_headers = { |
| 61 | "User-Agent": settings.user_agent, |
| 62 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", |
| 63 | "Accept-Language": "en-US,en;q=0.5", |
| 64 | } |
| 65 | if headers: |
| 66 | default_headers.update(headers) |
| 67 | |
| 68 | self._client = httpx.AsyncClient( |
| 69 | base_url=base_url or "", |
| 70 | timeout=httpx.Timeout(self.timeout), |
| 71 | headers=default_headers, |
| 72 | follow_redirects=True, |
| 73 | ) |
| 74 | |
| 75 | async def get( |
| 76 | self, |
| 77 | url: str, |
| 78 | params: dict[str, Any] | None = None, |
| 79 | headers: dict[str, str] | None = None, |
| 80 | ) -> httpx.Response: |
| 81 | """Perform rate-limited GET request.""" |
| 82 | await self.rate_limiter.acquire() |
| 83 | return await self._client.get(url, params=params, headers=headers) |
| 84 | |
| 85 | async def post( |
| 86 | self, |
| 87 | url: str, |
| 88 | data: dict[str, Any] | None = None, |
| 89 | json: dict[str, Any] | None = None, |
| 90 | headers: dict[str, str] | None = None, |
| 91 | ) -> httpx.Response: |
| 92 | """Perform rate-limited POST request.""" |
| 93 | await self.rate_limiter.acquire() |
| 94 | return await self._client.post(url, data=data, json=json, headers=headers) |
| 95 | |
| 96 | async def head( |
| 97 | self, |
| 98 | url: str, |
| 99 | headers: dict[str, str] | None = None, |
| 100 | ) -> httpx.Response: |
| 101 | """Perform rate-limited HEAD request.""" |
| 102 | await self.rate_limiter.acquire() |