Method indexAll

src/extraction/index.ts:1508–2094

* Index all files in the project

(
    onProgress?: (progress: IndexProgress) => void,
    signal?: AbortSignal,
    verbose?: boolean,
    // Writer-side backstop for deferred WAL checkpointing (#1231): returns
    // null in the normal case, or a promise to await (at this safe,
    // between-transactions boundary) when the WAL has outrun the off-thread
    // checkpointer past its hard cap. See db/wal-valve.ts.
    walBackpressure?: () => Promise<void> | null,
    // Fresh-DB store offload (perf): when set, per-file store bundles are
    // applied by a dedicated writer thread instead of the main thread. Only
    // passed for a COMPLETELY fresh database, where the main thread performs
    // no reads/writes during the parse loop, so one writer applying bundles
    // in file order preserves the #1015 determinism exactly.
    storeWriterOpts?: { dbPath: string; fastInit: boolean } | null
  )

   * Index all files in the project
   */
  async indexAll(
    onProgress?: (progress: IndexProgress) => void,
    signal?: AbortSignal,
    verbose?: boolean,
    // Writer-side backstop for deferred WAL checkpointing (#1231): returns
    // null in the normal case, or a promise to await (at this safe,
    // between-transactions boundary) when the WAL has outrun the off-thread
    // checkpointer past its hard cap. See db/wal-valve.ts.
    walBackpressure?: () => Promise<void> | null,
    // Fresh-DB store offload (perf): when set, per-file store bundles are
    // applied by a dedicated writer thread instead of the main thread. Only
    // passed for a COMPLETELY fresh database, where the main thread performs
    // no reads/writes during the parse loop, so one writer applying bundles
    // in file order preserves the #1015 determinism exactly.
    storeWriterOpts?: { dbPath: string; fastInit: boolean } | null
  ): Promise<IndexResult> {
    const tGrammar = Date.now();
    await initGrammars();
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] grammar-init: ${Date.now() - tGrammar}ms`);
    const startTime = Date.now();
    const errors: ExtractionError[] = [];
    let filesIndexed = 0;
    let filesSkipped = 0;
    let filesErrored = 0;
    let totalNodes = 0;
    let totalEdges = 0;

    // Custom extension → language overrides from the project's codegraph.json.
    // Threaded into language detection so custom-extension files load the right
    // grammar and store under the mapped language.
    const overrides = loadExtensionOverrides(this.rootDir);

    const log = verbose
      ? (msg: string) => { console.log(`[worker] ${msg}`); }
      : (_msg: string) => {};

    // Phase 1: Scan for files
    onProgress?.({
      phase: 'scanning',
      current: 0,
      total: 0,
    });

    // Phase attribution to stderr (same opt-in as the synthesis timings):
    // early-run 5-10s single stalls were observed on 95k-file repos but never
    // attributed — these labels settle scan vs framework-detect vs grammars.
    const tScan = Date.now();
    const files = await scanDirectoryAsync(this.rootDir, (current, file) => {
      onProgress?.({
        phase: 'scanning',
        current,
        total: 0,
        currentFile: file,
      });
    });
    if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`);

    // Detect frameworks once per indexAll run using the scanned file list.