Function llm_generate

fastdeploy/benchmarks/latency.py:99–100

()

    dummy_prompts = [{"prompt_token_ids": batch} for batch in dummy_prompt_token_ids.tolist()]

    def llm_generate():
        llm.generate(dummy_prompts, sampling_params=sampling_params, use_tqdm=False, stream=True)

    def run_to_completion():
        start_time = time.perf_counter()