Function detect_text_encoding

astrbot/core/computer/file_read_utils.py:170–204

(sample: bytes)

def detect_text_encoding(sample: bytes) -> str | None:
    if not sample:
        return "utf-8"

    if b"\x00" in sample and not sample.startswith(_UTF_BOMS):
        odd_bytes = sample[1::2]
        even_bytes = sample[0::2]
        odd_zero_ratio = odd_bytes.count(0) / max(len(odd_bytes), 1)
        even_zero_ratio = even_bytes.count(0) / max(len(even_bytes), 1)
        if odd_zero_ratio < 0.8 and even_zero_ratio < 0.8:
            return None

    for encoding in _TEXT_ENCODINGS:
        try:
            decoded = sample.decode(encoding)
        except UnicodeDecodeError as exc:
            # Probe samples can end in the middle of a multibyte sequence.
            # When the decode failure only happens at the sample tail, trim a few
            # bytes and retry so UTF-8 text is not misclassified as binary.
            if exc.start >= len(sample) - 4:
                decoded = ""
                for trim_bytes in range(1, min(4, len(sample)) + 1):
                    try:
                        decoded = sample[:-trim_bytes].decode(encoding)
                        break
                    except UnicodeDecodeError:
                        continue
                if not decoded:
                    continue
            else:
                continue
        if _looks_like_text(decoded):
            return encoding

    return None


def read_local_text_range_sync(