* Index all files in the project * * Uses a mutex to prevent concurrent indexing operations.
(options: IndexOptions = {})
| 653 | * Uses a mutex to prevent concurrent indexing operations. |
| 654 | */ |
| 655 | async indexAll(options: IndexOptions = {}): Promise<IndexResult> { |
| 656 | return this.indexMutex.withLock(async () => { |
| 657 | const startedAt = Date.now(); |
| 658 | try { |
| 659 | this.fileLock.acquire(); |
| 660 | } catch { |
| 661 | return { success: false, filesIndexed: 0, filesSkipped: 0, filesErrored: 0, nodesCreated: 0, edgesCreated: 0, errors: [{ message: 'Could not acquire file lock - another process may be indexing', severity: 'error' as const }], durationMs: 0 }; |
| 662 | } |
| 663 | // Defer WAL auto-checkpointing for the whole bulk run (#1231): the |
| 664 | // default 1000-page interval re-writes hot pages into the main DB file |
| 665 | // over and over — ~95% of all disk I/O during a bulk index, and a |
| 666 | // 19+min → 45s difference on HDD-class storage. The valve bounds WAL |
| 667 | // growth by backfilling PASSIVEly on a worker thread (never blocking |
| 668 | // the writer or the #850 watchdog heartbeat); runMaintenance below does |
| 669 | // the final fold-up before the interval is restored in the finally. |
| 670 | // Kill switch: CODEGRAPH_NO_WAL_DEFER=1. Non-WAL journal modes (some |
| 671 | // network filesystems) have no WAL to defer — skip. |
| 672 | // Fast-init: on a COMPLETELY fresh DB, trade crash-durability for speed |
| 673 | // during the bulk build (journal in memory, no fsync). Safe because the |
| 674 | // DB is disposable until the index completes — index_state stays |
| 675 | // 'indexing' and a crashed init is re-run from scratch; existing DBs |
| 676 | // (re-index/sync) never take this path. Kill switch: |
| 677 | // CODEGRAPH_NO_FAST_INIT=1 (same pattern as CODEGRAPH_NO_WAL_DEFER). |
| 678 | const freshDb = this.queries.getNodeAndEdgeCount().nodes === 0; |
| 679 | const fastInit = process.env.CODEGRAPH_NO_FAST_INIT !== '1' && freshDb; |
| 680 | if (fastInit) { |
| 681 | try { |
| 682 | this.db.getDb().pragma('journal_mode = MEMORY'); |
| 683 | this.db.getDb().pragma('synchronous = OFF'); |
| 684 | } catch { /* keep WAL */ } |
| 685 | } |
| 686 | const deferWal = !fastInit && process.env.CODEGRAPH_NO_WAL_DEFER !== '1' && this.db.getJournalMode() === 'wal'; |
| 687 | let walValve: WalCheckpointValve | null = null; |
| 688 | let priorAutocheckpoint = 1000; |
| 689 | // Set when the fastInit+pool path below defers autocheckpointing, so the |
| 690 | // finally knows to restore the interval on that path too. |
| 691 | let restoreAutocheckpoint = false; |
| 692 | if (deferWal) { |
| 693 | priorAutocheckpoint = this.db.getWalAutocheckpoint(); |
| 694 | this.db.setWalAutocheckpoint(0); |
| 695 | walValve = new WalCheckpointValve( |
| 696 | this.db, |
| 697 | resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB, this.db.getDbFileSizeBytes()), |
| 698 | undefined, |
| 699 | options.verbose ? (m) => console.log(`[wal-valve] ${m}`) : undefined |
| 700 | ); |
| 701 | walValve.start(); |
| 702 | } |
| 703 | try { |
| 704 | const gitState = this.orchestrator.beginGitIndexState(true); |
| 705 | const before = this.queries.getNodeAndEdgeCount(); |
| 706 | // Mark the index as in-flight BEFORE any writes: a run killed |
| 707 | // mid-index (OOM, SIGKILL, the #850 liveness watchdog) leaves this |
| 708 | // marker behind, so `codegraph status` can tell a truncated index |
| 709 | // from a completed one instead of silently serving partial results. |
| 710 | try { this.queries.setMetadata('index_state', 'indexing'); } catch { /* metadata is advisory */ } |
| 711 | // Segment vocabulary. On a fresh database it starts empty, and the node |
| 712 | // write path fills it as every file indexes below. Over an existing |