| 44 | |
| 45 | |
| 46 | def send_one_batch(base_url, max_concurrency, input_requests, disable_tqdm): |
| 47 | selected_percentile_metrics = ["s_itl"] |
| 48 | selected_percentiles = [] |
| 49 | # Run benchmark |
| 50 | results = asyncio.run( |
| 51 | benchmark( |
| 52 | backend="openai-chat", |
| 53 | api_url=f"{base_url}/v1/chat/completions", |
| 54 | base_url=base_url, |
| 55 | model_id="default", |
| 56 | model_name="default", |
| 57 | input_requests=input_requests, |
| 58 | hyper_parameters={}, |
| 59 | logprobs=None, |
| 60 | request_rate=float("inf"), |
| 61 | burstiness=1.0, |
| 62 | disable_tqdm=disable_tqdm, |
| 63 | profile=False, |
| 64 | selected_percentile_metrics=selected_percentile_metrics, |
| 65 | selected_percentiles=selected_percentiles, |
| 66 | ignore_eos=False, |
| 67 | goodput_config_dict=None, |
| 68 | max_concurrency=max_concurrency, |
| 69 | lora_modules=None, |
| 70 | extra_body=None, |
| 71 | ) |
| 72 | ) |
| 73 | |
| 74 | record = { |
| 75 | "mean_s_itl_ms": results["mean_s_itl_ms"], |
| 76 | } |
| 77 | |
| 78 | return record |
| 79 | |
| 80 | |
| 81 | def calculate_speedup(acceptance_rate, draft_token_step, t_ori, t_mtp): |