A wrapper around urllib.request.urlopen() which adds custom headers
(url, *args, **kwargs)
| 110 | return str(bs4.BeautifulSoup(data, "html5lib")) |
| 111 | |
| 112 | def urlopen(url, *args, **kwargs): |
| 113 | """A wrapper around urllib.request.urlopen() which adds custom headers""" |
| 114 | if isinstance(url, urllib.request.Request): |
| 115 | req = url |
| 116 | else: |
| 117 | req = urllib.request.Request(url) |
| 118 | req.add_header('User-Agent', _USER_AGENT) |
| 119 | return urllib.request.urlopen(req, *args, **kwargs) |
| 120 | |
| 121 | def build_opener(*args, **kwargs): |
| 122 | """A wrapper around urllib.request.build_opener() which adds custom headers""" |