Function validateQuestion

src/validate-questions.ts:60–219

(question: Question, schema: string)

// Research agent that validates a single question
async function validateQuestion(question: Question, schema: string): Promise<ValidationResult> {
  console.log(`\n${'='.repeat(60)}`);
  console.log(`Validating ${question.id}: ${question.question}`);
  console.log(`Original answer: ${question.reference_answer}`);
  console.log(`${'='.repeat(60)}`);

  // Step 1: Ask the model to generate SQL queries to validate
  const sqlPlanResponse = await generateText({
    model: VALIDATION_MODEL,
    system: `You are a data analyst validating reference answers for an eval dataset.
Given a question about GitHub data, generate 1-3 SQL queries that would help validate the reference answer.

DATABASE SCHEMA:
${schema}

EXAMPLE QUERIES (use these patterns):
-- Count issues per repo:
SELECT repos.full_name, COUNT(*) as issue_count
FROM repos JOIN issues ON repos.id = issues.repo_id
GROUP BY repos.id ORDER BY issue_count DESC LIMIT 5;

-- Find issues by text:
SELECT repos.full_name, issues.number, issues.title
FROM issues JOIN repos ON issues.repo_id = repos.id
WHERE issues.body LIKE '%keyword%';

-- Count by owner:
SELECT owner, COUNT(*) FROM repos GROUP BY owner ORDER BY COUNT(*) DESC LIMIT 10;

-- PRs with merge status:
SELECT repos.full_name, pulls.number, pulls.title, pulls.merged
FROM pulls JOIN repos ON pulls.repo_id = repos.id
WHERE pulls.merged = 1;

IMPORTANT RULES:
- ALWAYS use full table names (repos, issues, pulls, comments, events, users)
- NEVER use aliases like r, i, p - use repos, issues, pulls directly
- labels_json is a JSON array string like '["bug","enhancement"]'
- Use LIKE for text search in body/title fields
- merged column in pulls is 0/1 (not true/false)
- Filter bots with: author NOT LIKE '%[bot]%' AND author NOT LIKE '%bot'

Return ONLY valid SQL queries, one per line. No explanation.`,
    prompt: `Question: ${question.question}
Reference answer to validate: ${question.reference_answer}

Generate SQL queries to verify this answer:`,
    maxOutputTokens: 1000,
  });

  // Parse SQL queries - handle multi-line queries by splitting on semicolons
  const sqlQueries = sqlPlanResponse.text
    .split(';')
    .map((q) => q.trim().replace(/\n/g, ' ').replace(/\s+/g, ' '))
    .filter((q) => q.toUpperCase().startsWith('SELECT') || q.toUpperCase().startsWith('WITH'))
    .slice(0, 3);

  console.log(`\nGenerated ${sqlQueries.length} SQL queries`);