网页请求核心函数 是否使用代理应由上层处理
(url: str, cookies=None, ua: str = None, extra_headers=None, return_type: str = None, encoding: str = None,
retry: int = 3, timeout: int = G_DEFAULT_TIMEOUT, proxies=None, verify=None)
| 13 | |
| 14 | |
| 15 | def get(url: str, cookies=None, ua: str = None, extra_headers=None, return_type: str = None, encoding: str = None, |
| 16 | retry: int = 3, timeout: int = G_DEFAULT_TIMEOUT, proxies=None, verify=None): |
| 17 | """ |
| 18 | 网页请求核心函数 |
| 19 | |
| 20 | 是否使用代理应由上层处理 |
| 21 | """ |
| 22 | errors = "" |
| 23 | headers = {"User-Agent": ua or G_USER_AGENT} |
| 24 | if extra_headers != None: |
| 25 | headers.update(extra_headers) |
| 26 | for i in range(retry): |
| 27 | try: |
| 28 | result = requests.get(url, headers=headers, timeout=timeout, proxies=proxies, |
| 29 | verify=verify, cookies=cookies) |
| 30 | if return_type == "object": |
| 31 | return result |
| 32 | elif return_type == "content": |
| 33 | return result.content |
| 34 | else: |
| 35 | result.encoding = encoding or result.apparent_encoding |
| 36 | return result.text |
| 37 | except Exception as e: |
| 38 | if config.getInstance().debug(): |
| 39 | print(f"[-]Connect: {url} retry {i + 1}/{retry}") |
| 40 | errors = str(e) |
| 41 | if config.getInstance().debug(): |
| 42 | if "getaddrinfo failed" in errors: |
| 43 | print("[-]Connect Failed! Please Check your proxy config") |
| 44 | print("[-]" + errors) |
| 45 | else: |
| 46 | print("[-]" + errors) |
| 47 | print('[-]Connect Failed! Please check your Proxy or Network!') |
| 48 | raise Exception('Connect Failed') |
| 49 | |
| 50 | |
| 51 | def post(url: str, data: dict=None, files=None, cookies=None, ua: str=None, return_type: str=None, encoding: str=None, |