We default to UTF-8 if text too short, because the detection can return a random encoding leading to confusing results given the `charset_normalizer` version (< 2.0.5). >>> too_short = ']"foo"' >>> detected = from_bytes(too_short.encode()).best().encoding >>> detected '
(content: ContentBytes)
| 9 | |
| 10 | |
| 11 | def detect_encoding(content: ContentBytes) -> str: |
| 12 | """ |
| 13 | We default to UTF-8 if text too short, because the detection |
| 14 | can return a random encoding leading to confusing results |
| 15 | given the `charset_normalizer` version (< 2.0.5). |
| 16 | |
| 17 | >>> too_short = ']"foo"' |
| 18 | >>> detected = from_bytes(too_short.encode()).best().encoding |
| 19 | >>> detected |
| 20 | 'ascii' |
| 21 | >>> too_short.encode().decode(detected) |
| 22 | ']"foo"' |
| 23 | """ |
| 24 | encoding = UTF8 |
| 25 | if len(content) > TOO_SMALL_SEQUENCE: |
| 26 | match = from_bytes(bytes(content)).best() |
| 27 | if match: |
| 28 | encoding = match.encoding |
| 29 | return encoding |
| 30 | |
| 31 | |
| 32 | def smart_decode(content: ContentBytes, encoding: str) -> Tuple[str, str]: |