Make POST request with response size limiting.
(
self,
url: str,
data: Optional[Dict] = None,
json: Optional[Dict] = None,
headers: Optional[Dict] = None,
**kwargs
)
| 211 | return None |
| 212 | |
| 213 | def post( |
| 214 | self, |
| 215 | url: str, |
| 216 | data: Optional[Dict] = None, |
| 217 | json: Optional[Dict] = None, |
| 218 | headers: Optional[Dict] = None, |
| 219 | **kwargs |
| 220 | ) -> Optional[requests.Response]: |
| 221 | """Make POST request with response size limiting.""" |
| 222 | try: |
| 223 | if self.rate_limiter: |
| 224 | self.rate_limiter.acquire() |
| 225 | if self.jitter_max > 0: |
| 226 | time.sleep(random.uniform(self.jitter_min, self.jitter_max)) |
| 227 | if self.ua_rotation: |
| 228 | self.session.headers['User-Agent'] = random.choice(self.ua_pool) |
| 229 | if self.proxy_pool: |
| 230 | proxy = self.proxy_pool[self.proxy_index % len(self.proxy_pool)] |
| 231 | self.session.proxies = {'http': proxy, 'https': proxy} |
| 232 | self.proxy_index += 1 |
| 233 | response = self.session.post( |
| 234 | url, |
| 235 | data=data, |
| 236 | json=json, |
| 237 | headers=headers, |
| 238 | timeout=self.timeout, |
| 239 | verify=self.verify_ssl, |
| 240 | stream=True, # Enable streaming |
| 241 | **kwargs |
| 242 | ) |
| 243 | |
| 244 | # Check response size for POST too |
| 245 | content_length = response.headers.get('Content-Length') |
| 246 | if content_length and int(content_length) > self.max_response_size: |
| 247 | logger.warning( |
| 248 | f"POST response too large for {url}: {content_length} bytes. Skipping." |
| 249 | ) |
| 250 | response.close() |
| 251 | return None |
| 252 | |
| 253 | # Read with size limit and timeout protection |
| 254 | content = b'' |
| 255 | start_time = time.time() |
| 256 | read_timeout = self.timeout * 2 # Give extra time for reading |
| 257 | |
| 258 | for chunk in response.iter_content(chunk_size=8192): |
| 259 | # Check if reading is taking too long |
| 260 | if time.time() - start_time > read_timeout: |
| 261 | logger.warning( |
| 262 | f"POST response reading timeout exceeded for {url} after {read_timeout}s. Truncating." |
| 263 | ) |
| 264 | response.close() |
| 265 | if content: |
| 266 | response._content = content |
| 267 | return response |
| 268 | return None |
| 269 | |
| 270 | content += chunk |