Function main

scripts/noop-ablate.js:151–274

()

// ── Main ──────────────────────────────────────────────────────────────────────

function main() {
  if (!skillFilter) {
    console.error('Usage: node scripts/noop-ablate.js --skill <name> [--scenario <filter>] [--dry-run]');
    process.exit(1);
  }

  const skillFile = path.join(SKILLS_DIR, skillFilter, 'SKILL.md');
  if (!fs.existsSync(skillFile)) {
    console.error(`No SKILL.md for "${skillFilter}".`);
    process.exit(1);
  }

  const original = fs.readFileSync(skillFile, 'utf8');
  const rawCandidates = detectNoOps(original).map(c => ({ ...c, skill: skillFilter }));
  const verdicts = loadLatestVerdicts();
  const edits = rawCandidates.map(c => resolveEdit(c, verdicts)).filter(e => e.action !== 'keep');

  let scenarios = discoverScenarios().filter(s => s.skill === skillFilter && !s.skipExecute);
  if (scenarioFilter) scenarios = scenarios.filter(s => s.name.includes(scenarioFilter));

  if (rawCandidates.length === 0) {
    console.log(`\nNo Tier 1 candidates in ${skillFilter}. Nothing to ablate.\n`);
    process.exit(0);
  }
  if (edits.length === 0) {
    console.log(`\nAll candidates in ${skillFilter} were judged "keep". Nothing to ablate.\n`);
    process.exit(0);
  }
  if (scenarios.length === 0) {
    console.log(`\n${skillFilter} has no runnable benchmark scenarios - cannot ablate empirically.`);
    console.log('Add a scenario under skills/' + skillFilter + '/__benchmarks__/ to enable Tier 3.\n');
    process.exit(0);
  }

  // ── Run-count math (the cost story) ───────────────────────────────────────
  const S = scenarios.length;
  const K = edits.length;
  const bisectWorst = K > 1 ? S * Math.ceil(Math.log2(K)) * 2 : 0;
  const planCalls = S /*baseline*/ + S /*batch*/ + bisectWorst;

  console.log('\nNo-op Ablation (Tier 3, empirical)\n' + '='.repeat(44));
  console.log(`Skill:      ${skillFilter}`);
  console.log(`Candidates: ${K}  (${edits.map(e => `L${e.lineNo}:${e.action}`).join(', ')})`);
  console.log(`Scenarios:  ${S}  (${scenarios.map(s => s.name).join(', ')})`);
  console.log(`Plan:       ${S} baseline + ${S} batched ablation` + (K > 1 ? ` + ≤${bisectWorst} bisect` : '') + ` = ≤${planCalls} model call(s)`);
  console.log('Method:     compare assertion outcome vectors; same vector => behavior unchanged => no-op confirmed.');

  if (DRY_RUN) {
    console.log('\n[dry-run] No model calls made. Re-run without --dry-run to execute.\n');
    process.exit(0);
  }

  const cli = findClaudeCLI();
  if (!cli) {
    console.error('\nclaude CLI not found - cannot run ablation. Use --dry-run to see the plan.\n');
    process.exit(1);
  }