(
self, session_id: str | None, request: Request | None, proxy_tier: int | None
)
| 174 | return proxy_info.url if proxy_info else None |
| 175 | |
| 176 | async def _pick_url( |
| 177 | self, session_id: str | None, request: Request | None, proxy_tier: int | None |
| 178 | ) -> tuple[URL | None, int | None]: |
| 179 | if self._new_url_function: |
| 180 | try: |
| 181 | result = self._new_url_function(session_id, request) |
| 182 | if inspect.isawaitable(result): |
| 183 | result = await result |
| 184 | |
| 185 | return URL(str(result)) if result is not None else None, None |
| 186 | except Exception as e: |
| 187 | raise ValueError('The provided "new_url_function" did not return a valid URL') from e |
| 188 | |
| 189 | if self._proxy_tier_tracker: |
| 190 | if request is not None and proxy_tier is None: |
| 191 | hostname = URL(request.url).host |
| 192 | if hostname is None: |
| 193 | raise ValueError('The request URL does not have a hostname') |
| 194 | |
| 195 | if request.last_proxy_tier is not None: |
| 196 | self._proxy_tier_tracker.add_error(hostname, request.last_proxy_tier) |
| 197 | |
| 198 | proxy_tier = self._proxy_tier_tracker.predict_tier(hostname) |
| 199 | |
| 200 | request.last_proxy_tier = proxy_tier |
| 201 | request.forefront = True |
| 202 | |
| 203 | if proxy_tier is not None: |
| 204 | urls = self._proxy_tier_tracker.get_tier_urls(proxy_tier) |
| 205 | else: |
| 206 | urls = self._proxy_tier_tracker.all_urls |
| 207 | elif self._proxy_urls: |
| 208 | urls = self._proxy_urls |
| 209 | else: |
| 210 | raise RuntimeError('Invalid state') |
| 211 | |
| 212 | if session_id is None: |
| 213 | url = urls[self._next_custom_url_index % len(urls)] |
| 214 | self._next_custom_url_index += 1 |
| 215 | return url, proxy_tier |
| 216 | |
| 217 | if session_id not in self._used_proxy_urls: |
| 218 | self._used_proxy_urls[session_id] = urls[self._next_custom_url_index % len(urls)] |
| 219 | self._next_custom_url_index += 1 |
| 220 | |
| 221 | return self._used_proxy_urls[session_id], proxy_tier |
| 222 | |
| 223 | |
| 224 | class _ProxyTierTracker: |
no test coverage detected