Attempts to make a unicode URL usable with ``urllib/urllib2``. More specifically, it attempts to convert the unicode object ``url``, which is meant to represent a IRI, to an unicode object that, containing only ASCII characters, is a valid URI. This involves: * IDNA/Puny-e
(url, forceQuote=False)
| 4556 | |
| 4557 | @cachedmethod |
| 4558 | def asciifyUrl(url, forceQuote=False): |
| 4559 | """ |
| 4560 | Attempts to make a unicode URL usable with ``urllib/urllib2``. |
| 4561 | |
| 4562 | More specifically, it attempts to convert the unicode object ``url``, |
| 4563 | which is meant to represent a IRI, to an unicode object that, |
| 4564 | containing only ASCII characters, is a valid URI. This involves: |
| 4565 | |
| 4566 | * IDNA/Puny-encoding the domain name. |
| 4567 | * UTF8-quoting the path and querystring parts. |
| 4568 | |
| 4569 | See also RFC 3987. |
| 4570 | |
| 4571 | # Reference: http://blog.elsdoerfer.name/2008/12/12/opening-iris-in-python/ |
| 4572 | |
| 4573 | >>> asciifyUrl(u'http://www.\\u0161u\\u0107uraj.com') |
| 4574 | 'http://www.xn--uuraj-gxa24d.com' |
| 4575 | """ |
| 4576 | |
| 4577 | parts = _urllib.parse.urlsplit(url) |
| 4578 | if not all((parts.scheme, parts.netloc, parts.hostname)): |
| 4579 | # apparently not an url |
| 4580 | return getText(url) |
| 4581 | |
| 4582 | if all(char in string.printable for char in url): |
| 4583 | return getText(url) |
| 4584 | |
| 4585 | hostname = parts.hostname |
| 4586 | |
| 4587 | if isinstance(hostname, six.binary_type): |
| 4588 | hostname = getUnicode(hostname) |
| 4589 | |
| 4590 | # idna-encode domain |
| 4591 | try: |
| 4592 | hostname = hostname.encode("idna") |
| 4593 | except: |
| 4594 | hostname = hostname.encode("punycode") |
| 4595 | |
| 4596 | # UTF8-quote the other parts. We check each part individually if |
| 4597 | # if needs to be quoted - that should catch some additional user |
| 4598 | # errors, say for example an umlaut in the username even though |
| 4599 | # the path *is* already quoted. |
| 4600 | def quote(s, safe): |
| 4601 | s = s or '' |
| 4602 | # Triggers on non-ascii characters - another option would be: |
| 4603 | # _urllib.parse.quote(s.replace('%', '')) != s.replace('%', '') |
| 4604 | # which would trigger on all %-characters, e.g. "&". |
| 4605 | if getUnicode(s).encode("ascii", "replace") != s or forceQuote: |
| 4606 | s = _urllib.parse.quote(getBytes(s), safe=safe) |
| 4607 | return s |
| 4608 | |
| 4609 | username = quote(parts.username, '') |
| 4610 | password = quote(parts.password, safe='') |
| 4611 | path = quote(parts.path, safe='/') |
| 4612 | query = quote(parts.query, safe="&=") |
| 4613 | |
| 4614 | # put everything back together |
| 4615 | netloc = getText(hostname) |
no test coverage detected
searching dependent graphs…