(ctx, dataset_path: Optional[str], num_samples: int,
random_seed: int, system_prompt: Optional[str],
max_input_length: int, max_output_length: int)
| 123 | @click.pass_context |
| 124 | @staticmethod |
| 125 | def command(ctx, dataset_path: Optional[str], num_samples: int, |
| 126 | random_seed: int, system_prompt: Optional[str], |
| 127 | max_input_length: int, max_output_length: int) -> None: |
| 128 | llm: Union[LLM, PyTorchLLM] = ctx.obj |
| 129 | sampling_params = SamplingParams( |
| 130 | max_tokens=max_output_length, |
| 131 | truncate_prompt_tokens=max_input_length) |
| 132 | evaluator = JsonModeEval(dataset_path, |
| 133 | num_samples=num_samples, |
| 134 | random_seed=random_seed, |
| 135 | apply_chat_template=True, |
| 136 | system_prompt=system_prompt) |
| 137 | evaluator.evaluate(llm, sampling_params) |
| 138 | llm.shutdown() |
nothing calls this directly
no test coverage detected