(cfgPath *string)
| 259 | } |
| 260 | |
| 261 | func ledgerCmd(cfgPath *string) *cobra.Command { |
| 262 | var idsFile, output string |
| 263 | exportEval := &cobra.Command{ |
| 264 | Use: "export-eval [request_id ...]", |
| 265 | Short: "Build eval dataset JSON from ledger rows (uses ai_request_json)", |
| 266 | Long: "Loads each request_id from the configured ledger and writes a JSON array suitable for `infercore eval run --dataset`. Rows with selected_backend set include expected_backend for routing regression.", |
| 267 | RunE: func(cmd *cobra.Command, args []string) error { |
| 268 | cfg, err := config.Load(*cfgPath) |
| 269 | if err != nil { |
| 270 | return err |
| 271 | } |
| 272 | st, err := requests.NewFromConfig(cfg) |
| 273 | if err != nil { |
| 274 | return err |
| 275 | } |
| 276 | if st == nil { |
| 277 | return fmt.Errorf("ledger is disabled; set ledger.enabled=true in config") |
| 278 | } |
| 279 | defer st.Close() |
| 280 | |
| 281 | var ids []string |
| 282 | if strings.TrimSpace(idsFile) != "" { |
| 283 | f, err := os.Open(idsFile) |
| 284 | if err != nil { |
| 285 | return err |
| 286 | } |
| 287 | defer f.Close() |
| 288 | sc := bufio.NewScanner(f) |
| 289 | for sc.Scan() { |
| 290 | line := strings.TrimSpace(sc.Text()) |
| 291 | if line == "" || strings.HasPrefix(line, "#") { |
| 292 | continue |
| 293 | } |
| 294 | ids = append(ids, line) |
| 295 | } |
| 296 | if err := sc.Err(); err != nil { |
| 297 | return err |
| 298 | } |
| 299 | } |
| 300 | ids = append(ids, args...) |
| 301 | if len(ids) == 0 { |
| 302 | return fmt.Errorf("provide request_id arguments or --ids-file with one id per line") |
| 303 | } |
| 304 | |
| 305 | ctx := context.Background() |
| 306 | var items []eval.Item |
| 307 | var convErrs []error |
| 308 | for _, id := range ids { |
| 309 | row, err := st.GetRequest(ctx, id) |
| 310 | if err != nil { |
| 311 | convErrs = append(convErrs, fmt.Errorf("%s: %w", id, err)) |
| 312 | continue |
| 313 | } |
| 314 | it, err := eval.ItemFromRequestRow(row) |
| 315 | if err != nil { |
| 316 | convErrs = append(convErrs, err) |
| 317 | continue |
| 318 | } |
no test coverage detected