(args)
| 1358 | |
| 1359 | |
| 1360 | def cmd_eval(args): |
| 1361 | model_id = getattr(args, 'model_id', DEFAULT_MODEL_ID) |
| 1362 | |
| 1363 | if PROJECT_ROOT.parent.name != 'evals': |
| 1364 | print_color(RED, "Skipping internal eval checks: companion repo not found.") |
| 1365 | return 1 |
| 1366 | |
| 1367 | # Check if cactus library exists |
| 1368 | lib_path = PROJECT_ROOT / "cactus" / "build" / "libcactus.a" |
| 1369 | if not lib_path.exists(): |
| 1370 | print_color(RED, "Error: Cactus library not built. Run 'cactus build' first.") |
| 1371 | return 1 |
| 1372 | |
| 1373 | class DownloadArgs: |
| 1374 | pass |
| 1375 | |
| 1376 | dlargs = DownloadArgs() |
| 1377 | dlargs.model_id = model_id |
| 1378 | dlargs.precision = getattr(args, 'precision', 'INT4') |
| 1379 | dlargs.cache_dir = getattr(args, 'cache_dir', None) |
| 1380 | dlargs.token = getattr(args, 'token', None) |
| 1381 | dlargs.reconvert = getattr(args, 'reconvert', False) |
| 1382 | |
| 1383 | download_result = cmd_download(dlargs) |
| 1384 | if download_result != 0: |
| 1385 | return download_result |
| 1386 | |
| 1387 | weights_dir = get_effective_weights_dir(model_id, args) |
| 1388 | extra = getattr(args, 'extra_args', None) or [] |
| 1389 | |
| 1390 | def extra_has_flag(flag: str) -> bool: |
| 1391 | for a in extra: |
| 1392 | if a == flag or a.startswith(flag + "="): |
| 1393 | return True |
| 1394 | return False |
| 1395 | |
| 1396 | mode_flags = [] |
| 1397 | if getattr(args, 'tools', False): mode_flags.append('tools') |
| 1398 | if getattr(args, 'llm', False): mode_flags.append('llm') |
| 1399 | if getattr(args, 'stt', False): mode_flags.append('stt') |
| 1400 | if getattr(args, 'vlm', False): mode_flags.append('vlm') |
| 1401 | if getattr(args, 'embed', False): mode_flags.append('embed') |
| 1402 | |
| 1403 | if len(mode_flags) > 1: |
| 1404 | print_color(RED, f"Error: choose only one eval mode flag, got: {' '.join(mode_flags)}") |
| 1405 | return 1 |
| 1406 | |
| 1407 | mode = mode_flags[0] if mode_flags else "tools" |
| 1408 | repo_root = PROJECT_ROOT.parent # evals/ |
| 1409 | cwd = repo_root |
| 1410 | |
| 1411 | if mode == "tools": |
| 1412 | eval_runner = repo_root / "tool-evals" / "run_eval_berk.py" |
| 1413 | elif mode == "stt": |
| 1414 | eval_runner = repo_root / "speech-evals" / "speech_eval.py" |
| 1415 | elif mode == "llm": |
| 1416 | eval_runner = repo_root / "text-evals" / "perplexity_eval.py" |
| 1417 | elif mode == "vlm": |
no test coverage detected