(result)
| 28 | } |
| 29 | |
| 30 | export function buildMeasurementPlan(result) { |
| 31 | const hasObservedUsage = Boolean(result.observedUsage?.sampleCount); |
| 32 | const hasBudgetPressure = |
| 33 | ["heavy", "excessive"].includes(result.budgets.trigger_cost_tokens?.band) || |
| 34 | ["heavy", "excessive"].includes(result.budgets.invoke_cost_tokens?.band); |
| 35 | const hasCode = |
| 36 | hasMetric(result, "ts_file_count") || |
| 37 | hasMetric(result, "py_file_count") || |
| 38 | result.target.kind === "directory"; |
| 39 | |
| 40 | const toolsets = [ |
| 41 | createToolset({ |
| 42 | id: "token-usage-observer", |
| 43 | label: "Token Usage Observer", |
| 44 | priority: !hasObservedUsage || hasBudgetPressure ? "high" : "medium", |
| 45 | goal: "Measure how many tokens the skill or plugin actually burns in representative runs.", |
| 46 | why: |
| 47 | "Static estimates are useful guardrails, but observed usage is what tells you whether a real workflow is affordable and whether caching or reasoning changes the picture.", |
| 48 | signals: [ |
| 49 | "observed_usage_sample_count", |
| 50 | "observed_input_tokens_avg", |
| 51 | "observed_total_tokens_avg", |
| 52 | "estimate_vs_observed_input_ratio", |
| 53 | ], |
| 54 | evidenceSources: [ |
| 55 | "Responses API usage logs", |
| 56 | "Codex-like session exports", |
| 57 | "JSONL traces captured from local benchmarking harnesses", |
| 58 | ], |
| 59 | starterPack: { |
| 60 | manifestName: "token-usage-pack", |
| 61 | focus: "Validate sample size, cold-start versus warm-cache behavior, and estimate drift over time.", |
| 62 | }, |
| 63 | }), |
| 64 | createToolset({ |
| 65 | id: "task-outcome-scorecard", |
| 66 | label: "Task Outcome Scorecard", |
| 67 | priority: "high", |
| 68 | goal: "Measure whether the skill helps users finish the intended job with fewer retries and less cleanup.", |
| 69 | why: |
| 70 | "A low-token skill is still a miss if it fails the task, and a verbose skill may be worth it if it consistently improves first-pass success.", |
| 71 | signals: [ |
| 72 | "task_success_rate", |
| 73 | "first_pass_success_rate", |
| 74 | "retry_rate", |
| 75 | "human_override_rate", |
| 76 | ], |
| 77 | evidenceSources: [ |
| 78 | "Task run logs", |
| 79 | "Structured user acceptance checklist", |
| 80 | "Before/after comparison runs on the same prompts", |
| 81 | ], |
| 82 | starterPack: { |
| 83 | manifestName: "task-outcomes-pack", |
| 84 | focus: "Score success, retries, and manual intervention across a fixed prompt set.", |
| 85 | }, |
| 86 | }), |
| 87 | createToolset({ |
no test coverage detected