Function extractTextFromStream

src/tools/filesystem/pdf-read.ts:137–192

* Extract text from a PDF content stream. The stream contains a sequence of * operators in PostScript-ish syntax. We care about: * - (literal string) Tj → show literal * - Tj → show hex-encoded literal * - [array] TJ → show array of strings with kerning

(stream: string)

 * or MacRoman encoding) this works fine.
 */
function extractTextFromStream(stream: string): string {
  const out: string[] = [];

  // Process line-by-line-ish: scan for text operators
  // (literal) Tj | (literal) ' | (literal) "
  const singleStringRe = /\(((?:\\\)|\\\(|\\\\|[^()\\]|\\[0-9]{1,3}|\\[nrtbf])*)\)\s*(Tj|'|")/g;
  // <hex> Tj
  const hexStringRe = /<([0-9A-Fa-f\s]+)>\s*Tj/g;
  // [arrays] TJ
  const arrayRe = /\[([^\]]*)\]\s*TJ/g;
  // T* and ET = line/block end
  const newlineRe = /\bT\*\b/g;
  const blockEndRe = /\bET\b/g;

  // We want to walk the stream in order, mixing all operators. Easiest:
  // tokenize by scanning forward and detecting which kind of op is next.
  // For simplicity here we do passes and merge — this loses some ordering
  // precision but works for most documents.

  // Pass 1: literals
  let m: RegExpExecArray | null;
  const tokens: Array<{ pos: number; text: string }> = [];
  singleStringRe.lastIndex = 0;
  while ((m = singleStringRe.exec(stream))) {
    tokens.push({ pos: m.index, text: decodePdfLiteral(m[1]!) + (m[2] === "'" || m[2] === '"' ? '\n' : '') });
  }
  hexStringRe.lastIndex = 0;
  while ((m = hexStringRe.exec(stream))) {
    tokens.push({ pos: m.index, text: decodeHex(m[1]!) });
  }
  arrayRe.lastIndex = 0;
  while ((m = arrayRe.exec(stream))) {
    const arrContent = m[1]!;
    // Pull out (literals) inside the array; ignore numeric kerning offsets
    const inner: string[] = [];
    const innerRe = /\(((?:\\\)|\\\(|\\\\|[^()\\]|\\[0-9]{1,3}|\\[nrtbf])*)\)|<([0-9A-Fa-f\s]+)>/g;
    let n: RegExpExecArray | null;
    while ((n = innerRe.exec(arrContent))) {
      if (n[1] !== undefined) inner.push(decodePdfLiteral(n[1]));
      else if (n[2]) inner.push(decodeHex(n[2]));
    }
    tokens.push({ pos: m.index, text: inner.join('') });
  }
  newlineRe.lastIndex = 0;
  while ((m = newlineRe.exec(stream))) {
    tokens.push({ pos: m.index, text: '\n' });
  }
  blockEndRe.lastIndex = 0;
  while ((m = blockEndRe.exec(stream))) {
    tokens.push({ pos: m.index, text: '\n' });
  }

  tokens.sort((a, b) => a.pos - b.pos);
  for (const t of tokens) out.push(t.text);
  return out.join('');
}

function decodePdfLiteral(s: string): string {