| 40 | |
| 41 | // Build mapping from input IDs to agent:model combinations by reading workflow file |
| 42 | function buildInputMapping(): Map<string, { agent: string; model: string }> { |
| 43 | const workflowPath = new URL( |
| 44 | "../.github/workflows/compare-models.yml", |
| 45 | import.meta.url, |
| 46 | ); |
| 47 | const workflowContent = readFileSync(workflowPath, "utf8"); |
| 48 | const workflow = YAML.parse(workflowContent); |
| 49 | |
| 50 | const inputs = workflow?.on?.workflow_dispatch?.inputs as Record< |
| 51 | string, |
| 52 | WorkflowInputDefinition |
| 53 | >; |
| 54 | if (!inputs) { |
| 55 | throw new Error("No workflow_dispatch inputs found in workflow file"); |
| 56 | } |
| 57 | |
| 58 | const mapping = new Map<string, { agent: string; model: string }>(); |
| 59 | |
| 60 | for (const [inputId, inputDef] of Object.entries(inputs)) { |
| 61 | // Parse description which is in format "agent:model" |
| 62 | const description = inputDef.description; |
| 63 | const parts = description.split(":"); |
| 64 | if (parts.length === 2) { |
| 65 | const [agent, model] = parts; |
| 66 | mapping.set(inputId, { agent, model }); |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | return mapping; |
| 71 | } |
| 72 | |
| 73 | function loadDataset(): DatasetEntry[] { |
| 74 | const raw = readFileSync(new URL("../dataset.yaml", import.meta.url), "utf8"); |