MCPcopy Create free account
hub / github.com/MoonshotAI/kimi-code / decodeTextWithErrors

Function decodeTextWithErrors

packages/kaos/src/internal.ts:135–179  ·  view source on GitHub ↗
(
  data: Buffer,
  encoding: BufferEncoding,
  errors: 'strict' | 'replace' | 'ignore' = 'strict',
  ignoreBOM: boolean = false,
)

Source from the content-addressed store, hash-verified

133 * @internal
134 */
135export function decodeTextWithErrors(
136 data: Buffer,
137 encoding: BufferEncoding,
138 errors: 'strict' | 'replace' | 'ignore' = 'strict',
139 ignoreBOM: boolean = false,
140): string {
141 // Map Node's BufferEncoding names to Web TextDecoder labels where the two
142 // diverge. Only UTF-family encodings participate in the strict/replace/
143 // ignore dance; the others are lossless and use Buffer.toString directly.
144 let webLabel: string | undefined;
145 // eslint-disable-next-line typescript-eslint/switch-exhaustiveness-check
146 switch (encoding) {
147 case 'utf-8':
148 case 'utf8':
149 webLabel = 'utf-8';
150 break;
151 case 'utf16le':
152 case 'ucs2':
153 case 'ucs-2':
154 webLabel = 'utf-16le';
155 break;
156 default:
157 webLabel = undefined;
158 }
159
160 if (webLabel === undefined) {
161 // Non-UTF encodings (hex/base64/latin1/binary/ascii) are lossless byte↔
162 // character mappings; `errors` is meaningless for them. Return raw.
163 return data.toString(encoding);
164 }
165
166 if (errors === 'strict') {
167 return new TextDecoder(webLabel, { fatal: true, ignoreBOM }).decode(data);
168 }
169
170 // 'ignore' must skip invalid input bytes/code units, not delete every
171 // replacement character in the decoded output. A file can contain a valid
172 // U+FFFD, and Python preserves it under errors="ignore".
173 if (errors === 'ignore') {
174 return webLabel === 'utf-8' ? decodeUtf8Ignore(data) : decodeUtf16LeIgnore(data);
175 }
176
177 // 'replace' → substitute each invalid sequence with U+FFFD (default).
178 return new TextDecoder(webLabel, { fatal: false, ignoreBOM }).decode(data);
179}
180
181/**
182 * Convert a glob pattern segment (e.g. "*.txt", "file?.log") into a RegExp.

Callers 6

internal.test.tsFile · 0.90
readTextMethod · 0.90
readTextMethod · 0.90
readLinesMethod · 0.90
readTailLinesMethod · 0.90
_readUtf8LinesMethod · 0.90

Calls 3

decodeUtf8IgnoreFunction · 0.85
decodeUtf16LeIgnoreFunction · 0.85
toStringMethod · 0.65

Tested by

no test coverage detected