Perform a HTTP request and return decoded JSON data
(self, url, method='GET', headers=None, body=None)
| 586 | return json.loads(response_body) |
| 587 | |
| 588 | def fetch(self, url, method='GET', headers=None, body=None): |
| 589 | """Perform a HTTP request and return decoded JSON data""" |
| 590 | |
| 591 | # ##### PROXY & HEADERS ##### |
| 592 | request_headers = self.prepare_request_headers(headers) |
| 593 | # proxy-url |
| 594 | proxyUrl = self.check_proxy_url_settings(url, method, headers, body) |
| 595 | if proxyUrl is not None: |
| 596 | request_headers.update({'Origin': self.origin}) |
| 597 | url = proxyUrl + self.url_encoder_for_proxy_url(url) |
| 598 | # proxy agents |
| 599 | proxies = None # set default |
| 600 | httpProxy, httpsProxy, socksProxy = self.check_proxy_settings(url, method, headers, body) |
| 601 | if httpProxy: |
| 602 | proxies = {} |
| 603 | proxies['http'] = httpProxy |
| 604 | elif httpsProxy: |
| 605 | proxies = {} |
| 606 | proxies['https'] = httpsProxy |
| 607 | elif socksProxy: |
| 608 | proxies = {} |
| 609 | # https://stackoverflow.com/a/15661226/2377343 |
| 610 | proxies['http'] = socksProxy |
| 611 | proxies['https'] = socksProxy |
| 612 | proxyAgentSet = proxies is not None |
| 613 | self.check_conflicting_proxies(proxyAgentSet, proxyUrl) |
| 614 | # specifically for async-python, there is ".proxies" property maintained |
| 615 | if (self.proxies is not None): |
| 616 | if (proxyAgentSet or proxyUrl): |
| 617 | raise ExchangeError(self.id + ' you have conflicting proxy settings - use either .proxies or http(s)Proxy / socksProxy / proxyUrl') |
| 618 | proxies = self.proxies |
| 619 | # log |
| 620 | if self.verbose: |
| 621 | self.log("\nfetch Request:", self.id, method, url, "RequestHeaders:", request_headers, "RequestBody:", body) |
| 622 | self.logger.debug("%s %s, Request: %s %s", method, url, request_headers, body) |
| 623 | # end of proxies & headers |
| 624 | |
| 625 | request_body = body |
| 626 | content_type_key = None |
| 627 | files = None |
| 628 | for k, v in request_headers.items(): |
| 629 | lk = k.lower() |
| 630 | if lk == 'content-type': |
| 631 | if v == 'multipart/form-data': |
| 632 | content_type_key = k |
| 633 | files = () |
| 634 | for k, v in body.items(): |
| 635 | files += ((k, (None, v)), ) |
| 636 | body = None |
| 637 | break |
| 638 | else: |
| 639 | break |
| 640 | if files is not None: |
| 641 | # requests would handle it for multipart/form-data |
| 642 | del request_headers[content_type_key] |
| 643 | if body: |
| 644 | body = body.encode() |
| 645 |