Function createStreamParser

packages/lang-core/src/parser/parser.ts:438–627

(cat: ParamMap, rootName?: string)

}

export function createStreamParser(cat: ParamMap, rootName?: string): StreamParser {
  let buf = ""; // raw accumulated input (kept for set() diffing)
  // Preprocessed view of `buf` (fences + comments stripped, same as parse()'s
  // preprocess). The completed-statement scan runs over THIS, never the raw
  // buffer — so leading markdown prose / ```fences``` (e.g. an apostrophe in
  // "You're …" before the program) can't corrupt statement-boundary detection.
  let cleaned = "";
  let completedEnd = 0; // watermark: how far into `cleaned` is already completed
  const completedStmtMap = new Map<string, Statement>();

  let completedCount = 0;
  let firstId = "";

  function addStmt(text: string) {
    // `text` is sliced from `cleaned`, so it's already fence/comment-free.
    const t = text.trim();
    if (!t) return;
    for (const s of split(tokenize(t))) {
      const expr = parseExpression(s.tokens);
      const stmt = classifyStatement(s, expr);
      completedStmtMap.set(s.id, stmt);
      completedCount++;
      if (!firstId) firstId = s.id;
    }
  }

  // Recompute `cleaned` from `buf`. If the already-completed prefix is no longer
  // a prefix of the new cleaned text (e.g. an opening ```fence``` just appeared
  // and shifted the stripped output), the watermark + cache are stale, so reset
  // and re-scan. When the prefix is stable (the common streaming case) the cache
  // is kept, so a partial trailing statement never blanks already-completed ones.
  function refreshCleaned() {
    const next = preprocess(buf);
    if (!next.startsWith(cleaned.slice(0, completedEnd))) {
      completedEnd = 0;
      completedStmtMap.clear();
      completedCount = 0;
      firstId = "";
    }
    cleaned = next;
  }

  function scanNewCompleted(): number {
    let depth = 0,
      ternaryDepth = 0,
      inStr: false | '"' | "'" = false,
      esc = false;
    let stmtStart = completedEnd;

    for (let i = completedEnd; i < cleaned.length; i++) {
      const c = cleaned[i];
      if (esc) {
        esc = false;
        continue;
      }
      if (c === "\\" && inStr) {
        esc = true;
        continue;