| 31 | } |
| 32 | |
| 33 | int BenchmarkCommandHandler::Handle(const ParsedCommand& cmd) { |
| 34 | std::string model_name; |
| 35 | std::string config_path; |
| 36 | |
| 37 | // Options |
| 38 | model_name = CommandParser::GetOption(cmd, "model", ""); |
| 39 | config_path = CommandParser::GetOption(cmd, "config", ""); |
| 40 | auto repeat_str = CommandParser::GetOption(cmd, "repeat", ""); |
| 41 | auto warmup_str = CommandParser::GetOption(cmd, "warmup", ""); // currently unused |
| 42 | |
| 43 | // If positional provided, prefer it unless --model set |
| 44 | if (model_name.empty() && !cmd.arguments.empty()) { |
| 45 | model_name = cmd.arguments[0]; |
| 46 | } |
| 47 | |
| 48 | // Load application config |
| 49 | auto& config_mgr = ConfigManager::GetInstance(); |
| 50 | auto app_config = config_mgr.LoadConfig(); |
| 51 | |
| 52 | // Resolve config path similar to 'run' |
| 53 | if (config_path.empty()) { |
| 54 | if (model_name.empty()) { |
| 55 | if (!app_config.default_model.empty()) { |
| 56 | model_name = app_config.default_model; |
| 57 | LOG_INFO("Using default model: " + model_name); |
| 58 | } else { |
| 59 | UserInterface::ShowError("Model name required and no default model set", |
| 60 | "Set a default model with: mnncli config set default_model <model_name>"); |
| 61 | std::cout << MakeUsage(mnncli::GetSpec("benchmark")) << "\n"; |
| 62 | return 1; |
| 63 | } |
| 64 | } |
| 65 | config_path = FileUtils::GetConfigPath(model_name); |
| 66 | } else { |
| 67 | // Expand path and best-effort derive model name |
| 68 | config_path = FileUtils::ExpandTilde(config_path); |
| 69 | std::string config_dir = std::filesystem::path(config_path).parent_path().string(); |
| 70 | std::string config_filename = std::filesystem::path(config_dir).filename().string(); |
| 71 | if (!config_filename.empty()) { |
| 72 | model_name = config_filename; |
| 73 | } else if (model_name.empty()) { |
| 74 | model_name = "custom_model"; |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | if (config_path.empty()) { |
| 79 | UserInterface::ShowError("Config path is empty", "Unable to determine config path for model: " + model_name); |
| 80 | std::cout << MakeUsage(mnncli::GetSpec("benchmark")) << "\n"; |
| 81 | return 1; |
| 82 | } |
| 83 | |
| 84 | LOG_INFO("Benchmarking model: " + model_name); |
| 85 | LOG_INFO("Config path: " + config_path); |
| 86 | |
| 87 | // Create and load model |
| 88 | auto llm = LLMManager::CreateLLM(config_path, true); |
| 89 | |
| 90 | // Prepare benchmark options |