Method index

src/core/embeddings.ts:443–488

(docs: SearchDocument[], rootDir: string)

  private documents: SearchDocument[] = [];
  private vectors: number[][] = [];
  async index(docs: SearchDocument[], rootDir: string): Promise<void> {
    this.documents = docs;
    const cache = await loadCache(rootDir);
    const uncached: { idx: number; text: string; hash: string }[] = [];

    this.vectors = new Array(docs.length);

    for (let i = 0; i < docs.length; i++) {
      const doc = docs[i];
      const rawText = `${doc.header} ${doc.symbols.join(" ")} ${doc.content}`;
      const hash = hashContent(rawText);

      if (cache[doc.path]?.hash === hash) {
        this.vectors[i] = cache[doc.path].vector;
      } else {
        uncached.push({ idx: i, text: rawText, hash });
      }
    }

    if (uncached.length > 0) {
      const batchSize = getEmbeddingBatchSize();
      for (let b = 0; b < uncached.length; b += batchSize) {
        const batch = uncached.slice(b, b + batchSize);
        try {
          const embeddings = await fetchEmbedding(batch.map((u) => u.text));
          for (let j = 0; j < batch.length; j++) {
            this.vectors[batch[j].idx] = embeddings[j];
            cache[docs[batch[j].idx].path] = { hash: batch[j].hash, vector: embeddings[j] };
          }
        } catch (error) {
          if (!isContextLengthError(error)) throw error;
          for (const item of batch) {
            try {
              const [vector] = await fetchEmbedding(item.text);
              this.vectors[item.idx] = vector;
              cache[docs[item.idx].path] = { hash: item.hash, vector };
            } catch (itemError) {
              if (!isContextLengthError(itemError)) throw itemError;
              delete cache[docs[item.idx].path];
            }
          }
        }
      }
      await saveCache(rootDir, cache);
    }
  }

  async search(query: string, optionsOrTopK?: number | SearchQueryOptions): Promise<SearchResult[]> {
    const options = resolveSearchOptions(optionsOrTopK);