(url: str = None, cookies: dict = None, ua: str = None, return_type: str = None,
encoding: str = None)
| 118 | |
| 119 | # with keep-alive feature |
| 120 | def get_html_session(url: str = None, cookies: dict = None, ua: str = None, return_type: str = None, |
| 121 | encoding: str = None): |
| 122 | config_proxy = config.getInstance().proxy() |
| 123 | session = requests.Session() |
| 124 | if isinstance(cookies, dict) and len(cookies): |
| 125 | requests.utils.add_dict_to_cookiejar(session.cookies, cookies) |
| 126 | retries = Retry(total=config_proxy.retry, connect=config_proxy.retry, backoff_factor=1, |
| 127 | status_forcelist=[429, 500, 502, 503, 504]) |
| 128 | session.mount("https://", TimeoutHTTPAdapter(max_retries=retries, timeout=config_proxy.timeout)) |
| 129 | session.mount("http://", TimeoutHTTPAdapter(max_retries=retries, timeout=config_proxy.timeout)) |
| 130 | if config_proxy.enable: |
| 131 | session.verify = config.getInstance().cacert_file() |
| 132 | session.proxies = config_proxy.proxies() |
| 133 | headers = {"User-Agent": ua or G_USER_AGENT} |
| 134 | session.headers = headers |
| 135 | try: |
| 136 | if isinstance(url, str) and len(url): |
| 137 | result = session.get(str(url)) |
| 138 | else: # 空url参数直接返回可重用session对象，无需设置return_type |
| 139 | return session |
| 140 | if not result.ok: |
| 141 | return None |
| 142 | if return_type == "object": |
| 143 | return result |
| 144 | elif return_type == "content": |
| 145 | return result.content |
| 146 | elif return_type == "session": |
| 147 | return result, session |
| 148 | else: |
| 149 | result.encoding = encoding or "utf-8" |
| 150 | return result.text |
| 151 | except requests.exceptions.ProxyError: |
| 152 | print("[-]get_html_session() Proxy error! Please check your Proxy") |
| 153 | except requests.exceptions.RequestException: |
| 154 | pass |
| 155 | except Exception as e: |
| 156 | print(f"[-]get_html_session() failed. {e}") |
| 157 | return None |
| 158 | |
| 159 | |
| 160 | def get_html_by_browser(url: str = None, cookies: dict = None, ua: str = None, return_type: str = None, |
no test coverage detected