Encoding-aware text reader - handles cases like on Windows where a file is UTF-8, but default locale is windows-1252
(file: Path)
| 59 | |
| 60 | |
| 61 | def utf_read_text(file: Path) -> str: |
| 62 | """Encoding-aware text reader |
| 63 | - handles cases like on Windows where a file is UTF-8, but default locale is windows-1252 |
| 64 | """ |
| 65 | if ON_WINDOWS: |
| 66 | f_bytes = file.read_bytes() |
| 67 | f_enc: str = chardet.detect(f_bytes)["encoding"] |
| 68 | # NOTE - can just special case utf-8 files here? |
| 69 | def_enc = locale.getpreferredencoding() |
| 70 | log.debug(f"Default encoding is {def_enc}, file encoded as {f_enc}") |
| 71 | if def_enc.upper() != f_enc.upper(): |
| 72 | log.warning(f"Text file {file} encoded as {f_enc}, auto-converting") |
| 73 | return f_bytes.decode(encoding=f_enc) |
| 74 | else: |
| 75 | # for linux/macOS assume utf-8 |
| 76 | return file.read_text() |
| 77 | |
| 78 | |
| 79 | def dict_drop_empty(xs: t.Optional[t.Dict] = None, none_only: bool = False, **kwargs) -> t.Dict: |