Function legacyDetectEncoding

src/pkg/utils/encoding.ts:179–223

(data: Uint8Array)

 * Header charset, BOM, UTF null-patterns, and valid UTF-8 are handled before this runs.
 */
const legacyDetectEncoding = (data: Uint8Array): string => {
  const sample = createLegacyDetectionSample(data);
  const analysedResult = chardet.analyse(sample).sort((a, b) => b.confidence - a.confidence);
  let highestConfidence = -1;
  const results = [];
  let leastCharLen = Infinity;
  for (const entry of analysedResult) {
    const encoding = entry.name.toLowerCase();
    if (unicodeEncodings.has(encoding)) continue;
    let decodedText;
    try {
      // 验证检测到的编码是否有效
      decodedText = bytesDecode(encoding, sample);
    } catch (_e: any) {
      // ignored
    }
    if (!decodedText) continue;
    if (highestConfidence < 0) {
      highestConfidence = entry.confidence;
      if (highestConfidence > 90) return encoding;
    } else if (highestConfidence > 70 && entry.confidence < 30) {
      // 不考虑 confidence 过低的编码
      break;
    } else if (highestConfidence > 50 && entry.confidence < 20) {
      // 不考虑 confidence 过低的编码
      break;
    }
    // 当字元符少，不足以自动判断时，改用文本重复性测试
    const chars = new Set(decodedText);
    let charLen = chars.size;
    if (charLen > leastCharLen) continue;
    if (chars.has("\ufffd")) {
      // 发现 REPLACEMENT CHARACTER，每个替代符视为独立字符，并至少增加1
      const rplCharLen = decodedText.split("\ufffd").length - 1;
      charLen += Math.max(rplCharLen, 1);
    }
    results.push({
      encoding,
      charLen: charLen,
    });
    if (charLen < leastCharLen) leastCharLen = charLen;
  }
  const ret = results.find((e) => e.charLen === leastCharLen);
  return ret?.encoding || "windows-1252";
};

const detectBOM = (u8: Uint8Array): string | null => {
  // UTF-8 BOM