Fetch using requests library.
(url: str, timeout: int = 30)
| 153 | |
| 154 | |
| 155 | def fetch_with_requests(url: str, timeout: int = 30) -> str: |
| 156 | """Fetch using requests library.""" |
| 157 | headers = { |
| 158 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', |
| 159 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', |
| 160 | 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', |
| 161 | 'Accept-Encoding': 'identity', # Request uncompressed content |
| 162 | 'DNT': '1', |
| 163 | 'Connection': 'keep-alive', |
| 164 | 'Upgrade-Insecure-Requests': '1', |
| 165 | 'Cache-Control': 'max-age=0', |
| 166 | } |
| 167 | |
| 168 | try: |
| 169 | response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True) |
| 170 | response.raise_for_status() |
| 171 | |
| 172 | # Try to detect encoding |
| 173 | if response.encoding is None or response.encoding == 'ISO-8859-1': |
| 174 | response.encoding = response.apparent_encoding or 'utf-8' |
| 175 | |
| 176 | return response.text |
| 177 | |
| 178 | except requests.exceptions.RequestException as e: |
| 179 | return f"Error fetching URL: {str(e)}" |
| 180 | |
| 181 | |
| 182 | def fetch_with_urllib(url: str, timeout: int = 30) -> str: |