网页请求核心函数
(url, cookies: dict = None, ua: str = None, return_type: str = None, encoding: str = None, json_headers=None)
| 32 | |
| 33 | |
| 34 | def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None, encoding: str = None, json_headers=None): |
| 35 | """ |
| 36 | 网页请求核心函数 |
| 37 | """ |
| 38 | verify = config.getInstance().cacert_file() |
| 39 | config_proxy = config.getInstance().proxy() |
| 40 | errors = "" |
| 41 | |
| 42 | headers = {"User-Agent": ua or G_USER_AGENT} # noqa |
| 43 | if json_headers is not None: |
| 44 | headers.update(json_headers) |
| 45 | |
| 46 | for i in range(config_proxy.retry): |
| 47 | try: |
| 48 | if config_proxy.enable: |
| 49 | proxies = config_proxy.proxies() |
| 50 | result = requests.get(str(url), headers=headers, timeout=config_proxy.timeout, proxies=proxies, |
| 51 | verify=verify, |
| 52 | cookies=cookies) |
| 53 | else: |
| 54 | result = requests.get(str(url), headers=headers, timeout=config_proxy.timeout, cookies=cookies) |
| 55 | |
| 56 | if return_type == "object": |
| 57 | return result |
| 58 | elif return_type == "content": |
| 59 | return result.content |
| 60 | else: |
| 61 | result.encoding = encoding or result.apparent_encoding |
| 62 | return result.text |
| 63 | except Exception as e: |
| 64 | print("[-]Connect retry {}/{}".format(i + 1, config_proxy.retry)) |
| 65 | errors = str(e) |
| 66 | if "getaddrinfo failed" in errors: |
| 67 | print("[-]Connect Failed! Please Check your proxy config") |
| 68 | debug = config.getInstance().debug() |
| 69 | if debug: |
| 70 | print("[-]" + errors) |
| 71 | else: |
| 72 | print("[-]" + errors) |
| 73 | print('[-]Connect Failed! Please check your Proxy or Network!') |
| 74 | raise Exception('Connect Failed') |
| 75 | |
| 76 | |
| 77 | def post_html(url: str, query: dict, headers: dict = None) -> requests.Response: |
no test coverage detected