Method command

tensorrt_llm/evaluate/longbench_v2.py:760–793

(ctx, dataset_path: str, prompts_dir: Optional[str],
                num_samples: Optional[int], start_idx: int,
                difficulty: Optional[str], length: str, domain: Optional[str],
                cot: bool, no_context: bool, rag: int,
                output_dir: Optional[str], random_seed: int,
                apply_chat_template: bool, system_prompt: Optional[str],
                max_input_length: int, max_output_length: int,
                chat_template_kwargs: Optional[dict[str, Any]],
                temperature: float, top_p: float)

    @click.pass_context
    @staticmethod
    def command(ctx, dataset_path: str, prompts_dir: Optional[str],
                num_samples: Optional[int], start_idx: int,
                difficulty: Optional[str], length: str, domain: Optional[str],
                cot: bool, no_context: bool, rag: int,
                output_dir: Optional[str], random_seed: int,
                apply_chat_template: bool, system_prompt: Optional[str],
                max_input_length: int, max_output_length: int,
                chat_template_kwargs: Optional[dict[str, Any]],
                temperature: float, top_p: float) -> None:
        llm: Union[LLM, PyTorchLLM] = ctx.obj

        sampling_params = SamplingParams(max_tokens=max_output_length,
                                         temperature=temperature,
                                         top_p=top_p)

        evaluator = LongBenchV2(dataset_path=dataset_path,
                                prompts_dir=prompts_dir,
                                num_samples=num_samples,
                                start_idx=start_idx,
                                difficulty=difficulty,
                                length=length,
                                domain=domain,
                                cot=cot,
                                no_context=no_context,
                                rag=rag,
                                max_input_length=max_input_length,
                                output_dir=output_dir,
                                random_seed=random_seed,
                                apply_chat_template=apply_chat_template,
                                system_prompt=system_prompt,
                                chat_template_kwargs=chat_template_kwargs)

        evaluator.evaluate(llm, sampling_params)
        llm.shutdown()