Function forEachJsonLine

src/proxy/mailbox/store.js:128–213

(filePath, onRow, opts = {})

}

function forEachJsonLine(filePath, onRow, opts = {}) {
  const stats = { parsed: 0, corrupt: 0, overlong: 0 };
  if (!fs.existsSync(filePath)) return stats;

  const maxLineBytes = Math.max(1, Math.floor(Number(opts.maxLineBytes) || resolveMaxJsonlLineBytes()));
  const chunk = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES);
  let fd = null;
  let parts = [];
  let lineBytes = 0;
  let dropping = false;

  const resetLine = () => {
    parts = [];
    lineBytes = 0;
    dropping = false;
  };

  const appendSegment = (segment) => {
    if (!segment || segment.length === 0 || dropping) return;
    if (lineBytes + segment.length > maxLineBytes) {
      parts = [];
      lineBytes = 0;
      dropping = true;
      return;
    }
    parts.push(Buffer.from(segment));
    lineBytes += segment.length;
  };

  const finishLine = (hasNewline = false) => {
    const totalLineBytes = lineBytes + (hasNewline ? 1 : 0);
    if (dropping) {
      stats.overlong += 1;
      resetLine();
      return;
    }
    if (totalLineBytes > maxLineBytes) {
      stats.overlong += 1;
      resetLine();
      return;
    }
    if (lineBytes === 0) {
      resetLine();
      return;
    }
    const trimmed = Buffer.concat(parts, lineBytes).toString('utf8').trim();
    resetLine();
    if (!trimmed) return;
    try {
      onRow(JSON.parse(trimmed));
      stats.parsed += 1;
    } catch {
      stats.corrupt += 1;
    }
  };

  try {
    fd = fs.openSync(filePath, 'r');