| 2581 | bool retry) { |
| 2582 | uint64_t budget = (!cfg->max_tokens_set && tc->max_tokens) ? |
| 2583 | tc->max_tokens : (uint32_t)cfg->max_tokens; |
| 2584 | if (retry) budget *= 2; |
| 2585 | if (budget > EVAL_MAX_CONTEXT) budget = EVAL_MAX_CONTEXT; |
| 2586 | return (int)budget; |
| 2587 | } |
| 2588 | |
| 2589 | static int eval_max_generation_budget(const eval_config *cfg, |
| 2590 | const eval_case *cases, |
| 2591 | int ncases) { |
| 2592 | int max_budget = 0; |
| 2593 | for (int i = 0; i < ncases; i++) { |
| 2594 | int budget = eval_case_generation_budget(cfg, &cases[i], |
| 2595 | cfg->retry_incomplete); |
| 2596 | if (budget > max_budget) max_budget = budget; |
| 2597 | } |
| 2598 | return max_budget; |
| 2599 | } |
| 2600 | |
| 2601 | static int eval_max_prompt_tokens(ds4_engine *engine, |
| 2602 | const eval_config *cfg, |
| 2603 | const eval_case *cases, |
| 2604 | int ncases, |
| 2605 | int ctx_for_think_mode, |
| 2606 | int *max_case_out) |
| 2607 | { |
| 2608 | int max_prompt = 0; |
| 2609 | int max_case = -1; |
| 2610 | const ds4_think_mode think_mode = |
| 2611 | ds4_think_mode_for_context(cfg->think_mode, ctx_for_think_mode); |
| 2612 | |
| 2613 | for (int i = 0; i < ncases; i++) { |
| 2614 | char *question = build_question_prompt(&cases[i]); |
| 2615 | if (!question) { |
| 2616 | fprintf(stderr, "ds4-eval: failed to allocate prompt\n"); |
| 2617 | exit(1); |
| 2618 | } |
| 2619 | ds4_tokens prompt = {0}; |
| 2620 | ds4_encode_chat_prompt(engine, eval_system_prompt(), question, think_mode, &prompt); |
| 2621 | if (prompt.len > max_prompt) { |
| 2622 | max_prompt = prompt.len; |
| 2623 | max_case = i; |
| 2624 | } |
| 2625 | ds4_tokens_free(&prompt); |
| 2626 | free(question); |
| 2627 | } |
| 2628 | if (max_case_out) *max_case_out = max_case; |
| 2629 | return max_prompt; |
| 2630 | } |
| 2631 | |
| 2632 | static int eval_auto_context_size(ds4_engine *engine, |
| 2633 | eval_config *cfg, |
| 2634 | const eval_case *cases, |
| 2635 | int ncases, |
| 2636 | int max_generation_tokens, |
| 2637 | int *max_prompt_out, |
| 2638 | int *max_case_out) |
| 2639 | { |
| 2640 | int ctx = EVAL_MAX_CONTEXT; |
no test coverage detected