| 46 | |
| 47 | |
| 48 | class CurlAsyncHTTPClient(AsyncHTTPClient): |
| 49 | def initialize( # type: ignore |
| 50 | self, max_clients: int = 10, defaults: Optional[Dict[str, Any]] = None |
| 51 | ) -> None: |
| 52 | super().initialize(defaults=defaults) |
| 53 | # Typeshed is incomplete for CurlMulti, so just use Any for now. |
| 54 | self._multi = pycurl.CurlMulti() # type: Any |
| 55 | self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) |
| 56 | self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) |
| 57 | self._curls = [self._curl_create() for i in range(max_clients)] |
| 58 | self._free_list = self._curls[:] |
| 59 | self._requests = ( |
| 60 | collections.deque() |
| 61 | ) # type: Deque[Tuple[HTTPRequest, Callable[[HTTPResponse], None], float]] |
| 62 | self._fds = {} # type: Dict[int, int] |
| 63 | self._timeout = None # type: Optional[object] |
| 64 | |
| 65 | # libcurl has bugs that sometimes cause it to not report all |
| 66 | # relevant file descriptors and timeouts to TIMERFUNCTION/ |
| 67 | # SOCKETFUNCTION. Mitigate the effects of such bugs by |
| 68 | # forcing a periodic scan of all active requests. |
| 69 | self._force_timeout_callback = ioloop.PeriodicCallback( |
| 70 | self._handle_force_timeout, 1000 |
| 71 | ) |
| 72 | self._force_timeout_callback.start() |
| 73 | |
| 74 | # Work around a bug in libcurl 7.29.0: Some fields in the curl |
| 75 | # multi object are initialized lazily, and its destructor will |
| 76 | # segfault if it is destroyed without having been used. Add |
| 77 | # and remove a dummy handle to make sure everything is |
| 78 | # initialized. |
| 79 | dummy_curl_handle = pycurl.Curl() |
| 80 | self._multi.add_handle(dummy_curl_handle) |
| 81 | self._multi.remove_handle(dummy_curl_handle) |
| 82 | |
| 83 | def close(self) -> None: |
| 84 | self._force_timeout_callback.stop() |
| 85 | if self._timeout is not None: |
| 86 | self.io_loop.remove_timeout(self._timeout) |
| 87 | for curl in self._curls: |
| 88 | curl.close() |
| 89 | self._multi.close() |
| 90 | super().close() |
| 91 | |
| 92 | # Set below properties to None to reduce the reference count of current |
| 93 | # instance, because those properties hold some methods of current |
| 94 | # instance that will case circular reference. |
| 95 | self._force_timeout_callback = None # type: ignore |
| 96 | self._multi = None |
| 97 | |
| 98 | def fetch_impl( |
| 99 | self, request: HTTPRequest, callback: Callable[[HTTPResponse], None] |
| 100 | ) -> None: |
| 101 | self._requests.append((request, callback, self.io_loop.time())) |
| 102 | self._process_queue() |
| 103 | self._set_timeout(0) |
| 104 | |
| 105 | def _handle_socket(self, event: int, fd: int, multi: Any, data: bytes) -> None: |
no outgoing calls