Convert a URL to IDN notation
(url)
| 3028 | return _StringIO(url_file_stream_or_string) |
| 3029 | |
| 3030 | def _convert_to_idn(url): |
| 3031 | """Convert a URL to IDN notation""" |
| 3032 | # this function should only be called with a unicode string |
| 3033 | # strategy: if the host cannot be encoded in ascii, then |
| 3034 | # it'll be necessary to encode it in idn form |
| 3035 | parts = list(urlparse.urlsplit(url)) |
| 3036 | try: |
| 3037 | parts[1].encode('ascii') |
| 3038 | except UnicodeEncodeError: |
| 3039 | # the url needs to be converted to idn notation |
| 3040 | host = parts[1].rsplit(':', 1) |
| 3041 | newhost = [] |
| 3042 | port = u'' |
| 3043 | if len(host) == 2: |
| 3044 | port = host.pop() |
| 3045 | for h in host[0].split('.'): |
| 3046 | newhost.append(h.encode('idna').decode('utf-8')) |
| 3047 | parts[1] = '.'.join(newhost) |
| 3048 | if port: |
| 3049 | parts[1] += ':' + port |
| 3050 | return urlparse.urlunsplit(parts) |
| 3051 | else: |
| 3052 | return url |
| 3053 | |
| 3054 | def _build_urllib2_request(url, agent, etag, modified, referrer, auth, request_headers): |
| 3055 | request = urllib2.Request(url) |