Function preparePages

bin/gstack-memory-ingest.ts:1111–1201

* Prepare phase: walk sources, apply incremental + optional-secret-scan filters, * parse transcripts/artifacts into PageRecord, render bodies with * frontmatter. Returns the PreparedPage[] to stage + counts of files * filtered at each gate. * * Secret scanning policy (post 2026-05-10 perf revie

(
  args: CliArgs,
  ctx: WalkContext,
  state: IngestState,
)

 *   for users who want belt-and-suspenders.
 */
function preparePages(
  args: CliArgs,
  ctx: WalkContext,
  state: IngestState,
): {
  prepared: PreparedPage[];
  skippedSecret: number;
  skippedDedup: number;
  skippedUnattributed: number;
  parseFailed: number;
  partialPages: number;
} {
  const prepared: PreparedPage[] = [];
  let skippedSecret = 0;
  let skippedDedup = 0;
  let skippedUnattributed = 0;
  let parseFailed = 0;
  let partialPages = 0;

  for (const { path, type } of walkAllSources(ctx)) {
    if (args.limit !== null && prepared.length >= args.limit) break;

    if (args.mode === "incremental" && !fileChangedSinceState(path, state)) {
      skippedDedup++;
      continue;
    }

    // Optional belt-and-suspenders: when --scan-secrets is set, scan the
    // source file with gitleaks and skip dirty ones. Off by default
    // because gstack-brain-sync already gates the cross-machine boundary
    // and per-file gitleaks costs ~256ms/file (4-8 min on a real corpus).
    if (args.scanSecrets) {
      const scan = secretScanFile(path);
      if (scan.scanner === "gitleaks" && scan.findings.length > 0) {
        skippedSecret++;
        if (!args.quiet) {
          console.error(
            `[secret-scan match] ${path} (${scan.findings.length} finding${
              scan.findings.length === 1 ? "" : "s"
            }); skipped`,
          );
        }
        continue;
      }
    }

    let page: PageRecord;
    try {
      if (type === "transcript") {
        const session = parseTranscriptJsonl(path);
        if (!session) {
          parseFailed++;
          continue;
        }
        if (!args.includeUnattributed && !session.cwd) {
          skippedUnattributed++;
          continue;
        }