(self, url, progress = True, headers = None, expand_gzip = True)
| 227 | return self._decode(response) |
| 228 | |
| 229 | def get_bytes(self, url, progress = True, headers = None, expand_gzip = True): |
| 230 | response = self.open_url(url, headers) |
| 231 | if response is None: return None |
| 232 | try: total_size = int(response.headers['Content-Length']) |
| 233 | except: total_size = -1 |
| 234 | chunk_so_far = b"" |
| 235 | packets = queue = process = None |
| 236 | if progress: |
| 237 | # Make sure our vars are initialized |
| 238 | packets = [] if progress else None |
| 239 | queue = multiprocessing.Queue() |
| 240 | # Create the multiprocess and start it |
| 241 | process = multiprocessing.Process( |
| 242 | target=_process_hook, |
| 243 | args=(queue,total_size) |
| 244 | ) |
| 245 | process.daemon = True |
| 246 | # Filthy hack for earlier python versions on Windows |
| 247 | if os.name == "nt" and hasattr(multiprocessing,"forking"): |
| 248 | self._update_main_name() |
| 249 | process.start() |
| 250 | try: |
| 251 | chunk_size = self.chunk or 1024 |
| 252 | auto_chunk_size = not self.chunk |
| 253 | while True: |
| 254 | t = time.perf_counter() |
| 255 | chunk = response.read(chunk_size) |
| 256 | chunk_time = time.perf_counter()-t |
| 257 | if progress: |
| 258 | # Add our items to the queue |
| 259 | queue.put((time.time(),len(chunk))) |
| 260 | if not chunk: break |
| 261 | chunk_so_far += chunk |
| 262 | if auto_chunk_size: |
| 263 | # Adjust our chunk size based on the internet speed at our defined rate |
| 264 | chunk_rate = int(len(chunk) / chunk_time * self.chunk_rate) |
| 265 | chunk_change_max = round(chunk_size * self.chunk_growth) |
| 266 | chunk_rate_clamped = min(max(self.min_chunk, chunk_rate), chunk_change_max) |
| 267 | chunk_size = min(chunk_rate_clamped, self.max_chunk) |
| 268 | finally: |
| 269 | # Close the response whenever we're done |
| 270 | response.close() |
| 271 | if expand_gzip and response.headers.get("Content-Encoding","unknown").lower() == "gzip": |
| 272 | fileobj = BytesIO(chunk_so_far) |
| 273 | gfile = gzip.GzipFile(fileobj=fileobj) |
| 274 | return gfile.read() |
| 275 | if progress: |
| 276 | # Finalize the queue and wait |
| 277 | queue.put("DONE") |
| 278 | process.join() |
| 279 | return chunk_so_far |
| 280 | |
| 281 | def stream_to_file(self, url, file_path, progress = True, headers = None, ensure_size_if_present = True, allow_resume = False): |
| 282 | response = self.open_url(url, headers) |
no test coverage detected