| 32 | } |
| 33 | |
| 34 | int ModelRunner::EvalPrompts(const std::vector<std::string>& prompts) { |
| 35 | int prompt_len = 0; |
| 36 | int decode_len = 0; |
| 37 | int64_t vision_time = 0; |
| 38 | int64_t audio_time = 0; |
| 39 | int64_t prefill_time = 0; |
| 40 | int64_t decode_time = 0; |
| 41 | int64_t sample_time = 0; |
| 42 | |
| 43 | for (int i = 0; i < prompts.size(); i++) { |
| 44 | const auto& prompt = prompts[i]; |
| 45 | if (prompt.substr(0, 1) == "#") { |
| 46 | continue; |
| 47 | } |
| 48 | |
| 49 | ProcessPrompt(prompt, &std::cout); |
| 50 | auto context = llm_->getContext(); |
| 51 | prompt_len += context->prompt_len; |
| 52 | decode_len += context->gen_seq_len; |
| 53 | vision_time += context->vision_us; |
| 54 | audio_time += context->audio_us; |
| 55 | prefill_time += context->prefill_us; |
| 56 | decode_time += context->decode_us; |
| 57 | sample_time += context->sample_us; |
| 58 | } |
| 59 | |
| 60 | ShowPerformanceMetrics(prompt_len, decode_len, vision_time, audio_time, |
| 61 | prefill_time, decode_time, sample_time); |
| 62 | |
| 63 | return 0; |
| 64 | } |
| 65 | |
| 66 | int ModelRunner::EvalFile(const std::string& prompt_file) { |
| 67 | std::cout << "Reading prompts from: " << prompt_file << "\n"; |
no test coverage detected