Method indexAll

src/index.ts:471–750

* Index all files in the project * * Uses a mutex to prevent concurrent indexing operations.

(options: IndexOptions = {})

   * Uses a mutex to prevent concurrent indexing operations.
   */
  async indexAll(options: IndexOptions = {}): Promise<IndexResult> {
    return this.indexMutex.withLock(async () => {
      try {
        this.fileLock.acquire();
      } catch {
        return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 };
      }
      // Defer WAL auto-checkpointing for the whole bulk run (#1231): the
      // default 1000-page interval re-writes hot pages into the main DB file
      // over and over — ~95% of all disk I/O during a bulk index, and a
      // 19+min → 45s difference on HDD-class storage. The valve bounds WAL
      // growth by backfilling PASSIVEly on a worker thread (never blocking
      // the writer or the #850 watchdog heartbeat); runMaintenance below does
      // the final fold-up before the interval is restored in the finally.
      // Kill switch: CODEGRAPH_NO_WAL_DEFER=1. Non-WAL journal modes (some
      // network filesystems) have no WAL to defer — skip.
      // Fast-init: on a COMPLETELY fresh DB, trade crash-durability for speed
      // during the bulk build (journal in memory, no fsync). Safe because the
      // DB is disposable until the index completes — index_state stays
      // 'indexing' and a crashed init is re-run from scratch; existing DBs
      // (re-index/sync) never take this path. Kill switch:
      // CODEGRAPH_NO_FAST_INIT=1 (same pattern as CODEGRAPH_NO_WAL_DEFER).
      const freshDb = this.queries.getNodeAndEdgeCount().nodes === 0;
      const fastInit = process.env.CODEGRAPH_NO_FAST_INIT !== '1' && freshDb;
      if (fastInit) {
        try {
          this.db.getDb().pragma('journal_mode = MEMORY');
          this.db.getDb().pragma('synchronous = OFF');
        } catch { /* keep WAL */ }
      }
      const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal';
      let walValve: WalCheckpointValve | null = null;
      let priorAutocheckpoint = 1000;
      // Set when the fastInit+pool path below defers autocheckpointing, so the
      // finally knows to restore the interval on that path too.
      let restoreAutocheckpoint = false;
      if (deferWal) {
        priorAutocheckpoint = this.db.getWalAutocheckpoint();
        this.db.setWalAutocheckpoint(0);
        walValve = new WalCheckpointValve(
          this.db,
          resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()),
          undefined,
          options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined
        );
        walValve.start();
      }
      try {
        const before = this.queries.getNodeAndEdgeCount();
        // Mark the index as in-flight BEFORE any writes: a run killed
        // mid-index (OOM, SIGKILL, the #850 liveness watchdog) leaves this
        // marker behind, so `codegraph status` can tell a truncated index
        // from a completed one instead of silently serving partial results.
        try { this.queries.setMetadata('index_state', 'indexing'); } catch { /* metadata is advisory */ }
        // Segment vocabulary starts empty and is repopulated by the node write
        // path as every file (re-)indexes below — so a full index is also the
        // orphan-cleanup pass for names deleted since the last one.
        try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ }