Function selectRelevantPassages

src/tools/web/extract-select.ts:79–124

(text: string, opts: { query?: string; budget: number })

 * query, fall back to the head+tail window. PURE.
 */
export function selectRelevantPassages(text: string, opts: { query?: string; budget: number }): ExtractSelection {
  const fullLength = text.length;
  const budget = Math.max(200, opts.budget);
  if (fullLength <= budget) return { content: text, mode: 'whole', fullLength, keptChars: fullLength, omittedChars: 0 };

  const query = (opts.query ?? '').trim();
  const passages = splitPassages(text);
  const qv = query ? termFreq(semanticTokens(query)) : null;
  if (!qv || qv.size === 0 || passages.length <= 1) {
    const ht = headTailWindow(text, budget);
    return { content: ht.content, mode: 'head-tail', fullLength, keptChars: ht.keptChars, omittedChars: fullLength - ht.keptChars };
  }

  // Score every passage; always keep passage 0 (title/lede) as context anchor.
  const scored = passages.map((p, i) => ({ i, p, score: cosineSim(qv, termFreq(semanticTokens(p.text))) }));
  const chosen = new Set<number>([0]);
  let used = passages[0]!.text.length;
  for (const s of scored.filter(s => s.i !== 0 && s.score > 0).sort((a, b) => b.score - a.score)) {
    if (used + s.p.text.length + 2 > budget) continue;      // skip; a smaller relevant one may still fit
    chosen.add(s.i); used += s.p.text.length + 2;
  }
  // If the query matched nothing beyond the anchor, positional is the safer bet.
  if (chosen.size === 1 && passages.length > 1) {
    const ht = headTailWindow(text, budget);
    return { content: ht.content, mode: 'head-tail', fullLength, keptChars: ht.keptChars, omittedChars: fullLength - ht.keptChars };
  }

  // Emit chosen passages in document order, marking gaps between non-adjacent selections.
  const order = [...chosen].sort((a, b) => a - b);
  const chunks: string[] = [];
  let kept = 0; let prev = -1;
  for (const idx of order) {
    if (prev >= 0 && idx > prev + 1) {
      const gap = passages.slice(prev + 1, idx).reduce((a, p) => a + p.text.length, 0);
      chunks.push(`[… ${gap} chars omitted (less relevant) …]`);
    }
    chunks.push(passages[idx]!.text);
    kept += passages[idx]!.text.length;
    prev = idx;
  }
  if (prev < passages.length - 1) {
    const gap = passages.slice(prev + 1).reduce((a, p) => a + p.text.length, 0);
    chunks.push(`[… ${gap} chars omitted (less relevant) …]`);
  }
  return { content: chunks.join('\n\n'), mode: 'semantic', fullLength, keptChars: kept, omittedChars: fullLength - kept };
}