Function searchDocs

apps/docs/app/api/chat/route.ts:216–276

* Retrieve candidate chunks for grounding, mirroring the site search route's * hybrid strategy: Postgres full-text keyword search for every locale, plus * vector similarity (thresholded) for English — fused by reciprocal rank so a * page found by either signal can ground the answer.

(query: string, locale: string)

 * page found by either signal can ground the answer.
 */
async function searchDocs(query: string, locale: string) {
  const tsConfig = TS_CONFIG[locale] ?? 'simple'

  // Each retrieval path is best-effort and independent: a failure in one still
  // lets the other ground the answer (both empty just yields no grounding).
  let keywordRows: SearchRow[] = []
  try {
    keywordRows = await db
      .select(SEARCH_COLUMNS)
      .from(docsEmbeddings)
      .where(
        sql`${docsEmbeddings.chunkTextTsv} @@ plainto_tsquery(${tsConfig}, ${query}) and ${localeFilter(locale)}`
      )
      .orderBy(
        sql`ts_rank(${docsEmbeddings.chunkTextTsv}, plainto_tsquery(${tsConfig}, ${query})) DESC`
      )
      .limit(SEARCH_CANDIDATES)
  } catch (error) {
    console.error('Ask AI keyword search failed:', error)
  }

  let vectorRows: SearchRow[] = []
  if (locale === DEFAULT_LOCALE) {
    // Vector retrieval (embedding call + pgvector query) is best-effort: if it
    // fails, fall back to the keyword rows already fetched rather than losing all
    // grounding for the turn.
    try {
      const embedding = await generateSearchEmbedding(query)
      const vectorLiteral = JSON.stringify(embedding)
      vectorRows = await db
        .select(SEARCH_COLUMNS)
        .from(docsEmbeddings)
        .where(
          sql`1 - (${docsEmbeddings.embedding} <=> ${vectorLiteral}::vector) >= ${SIMILARITY_THRESHOLD} and ${localeFilter(locale)}`
        )
        .orderBy(sql`${docsEmbeddings.embedding} <=> ${vectorLiteral}::vector`)
        .limit(SEARCH_CANDIDATES)
    } catch (error) {
      console.error('Ask AI vector search failed; using keyword results only:', error)
    }
  }

  // Reciprocal rank fusion across the two rankings, deduped by chunk.
  const scores = new Map<string, number>()
  const rowById = new Map<string, SearchRow>()
  for (const list of [vectorRows, keywordRows]) {
    list.forEach((row, index) => {
      scores.set(row.chunkId, (scores.get(row.chunkId) ?? 0) + 1 / (RRF_K + index + 1))
      if (!rowById.has(row.chunkId)) rowById.set(row.chunkId, row)
    })
  }

  return [...rowById.values()]
    .sort((a, b) => (scores.get(b.chunkId) ?? 0) - (scores.get(a.chunkId) ?? 0))
    .slice(0, SEARCH_LIMIT)
    .map((row) => ({
      title: row.title,
      url: row.url,