Function retrieveRelevantFiles

src/context/retrieval.ts:434–530

(
  cwd: string,
  query: string,
  opts: RetrieveOptions = {},
)

 * MUST treat null as "skip silently", never as an error.
 */
export async function retrieveRelevantFiles(
  cwd: string,
  query: string,
  opts: RetrieveOptions = {},
): Promise<RankedFile[] | null> {
  const baseUrl = opts.baseUrl ?? process.env.OLLAMA_BASE_URL ?? 'http://localhost:11434';
  const model = opts.embeddingModel ?? 'nomic-embed-text';
  const topKChunks = opts.topKChunks ?? 24;
  const maxFiles = opts.maxFiles ?? 6;

  try {
    if (!(await ollamaReachable(baseUrl, opts.signal))) return null;

    const idxPath = indexPath(cwd, model);
    let idx = await loadIndex(idxPath);

    if (!idx) {
      if (!opts.buildIfMissing) {
        // No JSON index — but a SQLite index might still exist. Probe it.
        try {
          const { openSqliteIndex } = await import('./sqlite-index.js');
          const probe = await openSqliteIndex(cwd, model);
          if (probe) { probe.db.close(); }
          else return null;
        } catch { return null; }
      } else {
        idx = await buildIndex(cwd, model, baseUrl, opts.maxFilesToIndex ?? 1500, opts.signal, () => {});
        await persistIndex(cwd, model, idx);
      }
    }

    const [queryEmbedding] = await ollamaEmbed(baseUrl, model, [query], opts.signal);
    if (!queryEmbedding) return null;

    // Hybrid (BM25 + semantic) ranking over the persisted index (SQLite preferred).
    const scored = await searchPersisted(cwd, model, query, queryEmbedding, topKChunks, idx);
    if (scored.length === 0) return null;

    // Stage-2 cross-encoder rerank (optional). The bi-encoder gives us a fast,
    // wide candidate set; the cross-encoder re-scores query+doc jointly for true
    // relevance, then we narrow to maxFiles. If no reranker is reachable this is
    // a no-op and we keep the bi-encoder order.
    let ranked: RankedFile[];
    if (opts.rerank) {
      const candPool = opts.rerankCandidates ?? 40;
      const wideFiles = aggregateToFiles(scored, candPool); // wide net first
      try {
        const { crossEncoderRerank } = await import('./reranker.js');
        const reranked = await crossEncoderRerank(
          query,
          wideFiles.map(f => ({
            id: f.file,
            // Feed the cross-encoder the REAL code (prefixed with the path for light
            // context). Falling back to the filename+line string only when a candidate
            // has no chunk text — otherwise the reranker scores on path tokens alone,
            // which is what made its scores degenerate.
            text: f.text && f.text.trim()
              ? `${f.file}\n${f.text}`