Function _read_text_with_encoding

src/command_system/input_processing.py:309–355

Read ``path`` as text, picking a decoder from any leading BOM. Falls back to utf-8 with ``errors="replace"`` for the common case (no BOM). When a BOM is present we use the codec it implies (with ``errors="replace"`` so a malformed trailing byte doesn't bring down the whole read).

(path: str)

def _read_text_with_encoding(path: str) -> str | None:
    """Read ``path`` as text, picking a decoder from any leading BOM.

    Falls back to utf-8 with ``errors="replace"`` for the common case
    (no BOM). When a BOM is present we use the codec it implies (with
    ``errors="replace"`` so a malformed trailing byte doesn't bring
    down the whole read).

    Post-decode garbage check: if the decoded text is mostly U+FFFD
    replacement characters, returns ``None`` so the caller drops the
    attachment instead of inlining mojibake. This closes the adversarial
    case where a binary blob starts with `\\xff\\xfe` (fake UTF-16 BOM)
    and an extension we haven't enumerated — without the check, the
    BOM short-circuit would route the blob to the text branch and the
    `errors="replace"` decode would land replacement-char mojibake in
    the prompt (same failure family as Bug A even though no NULs leak).

    Returns ``None`` on OSError or on garbled-decode so the caller
    matches the existing "drop this attachment silently" behaviour for
    unreadable files. Callers should still emit a binary-style reminder
    via the @-mention `binary` attachment kind when this returns None
    for a non-OSError reason — see ``expand_at_mentions``.
    """
    try:
        with open(path, "rb") as fh:
            head = fh.read(4)
    except OSError:
        return None
    encoding = _detect_bom_encoding(head) or "utf-8"
    try:
        with open(path, "r", encoding=encoding, errors="replace") as fh:
            data = fh.read()
    except OSError:
        return None
    except (LookupError, UnicodeError):
        # Pathological codec name from a future BOM check, or a UTF-32
        # decode failure with errors=replace (rare but theoretically
        # possible). Fall back to a raw utf-8 replacement read so the
        # path still produces *some* text rather than a silent drop.
        try:
            with open(path, "r", encoding="utf-8", errors="replace") as fh:
                data = fh.read()
        except OSError:
            return None
    if _decoded_text_looks_garbled(data):
        return None
    return data


def _binary_hint_for_ext(ext: str) -> str: