Function classifyDomainsWithLlm

src/bookmark-classify-llm.ts:285–356

(
  options: { engine: ResolvedEngine; all?: boolean; onBatch?: (done: number, total: number) => void },
)

}

export async function classifyDomainsWithLlm(
  options: { engine: ResolvedEngine; all?: boolean; onBatch?: (done: number, total: number) => void },
): Promise<LlmClassifyResult> {
  const { engine } = options;

  const dbPath = twitterBookmarksIndexPath();
  const db = await openDb(dbPath);

  // Ensure domain columns exist (migration from schema v2)
  try { db.run('ALTER TABLE bookmarks ADD COLUMN domains TEXT'); } catch { /* already exists */ }
  try { db.run('ALTER TABLE bookmarks ADD COLUMN primary_domain TEXT'); } catch { /* already exists */ }

  try {
    const where = options.all
      ? '1=1'
      : 'primary_domain IS NULL';
    const rows = db.exec(
      `SELECT id, text, author_handle, categories FROM bookmarks
       WHERE ${where} ORDER BY RANDOM()`
    );

    if (!rows.length || !rows[0].values.length) {
      return { engine: engine.name, totalUnclassified: 0, classified: 0, failed: 0, batches: 0 };
    }

    const bookmarks: DomainBookmark[] = rows[0].values.map(r => ({
      id: r[0] as string,
      text: r[1] as string,
      authorHandle: r[2] as string | null,
      categories: r[3] as string | null,
    }));

    const total = bookmarks.length;
    let classified = 0;
    let failed = 0;
    let batchCount = 0;

    for (let i = 0; i < bookmarks.length; i += BATCH_SIZE) {
      const batch = bookmarks.slice(i, i + BATCH_SIZE);
      const batchIds = new Set(batch.map(b => b.id));
      batchCount++;

      options.onBatch?.(i, total);

      try {
        const prompt = buildDomainPrompt(batch);
        const raw = invokeEngine(engine, prompt);
        // Reuse the same parse logic — structure is identical
        const results = parseResponse(raw, batchIds);

        const stmt = db.prepare(
          `UPDATE bookmarks SET domains = ?, primary_domain = ? WHERE id = ?`
        );
        for (const r of results) {
          stmt.run([r.categories.join(','), r.primary, r.id]);
        }
        stmt.free();