| 380 | |
| 381 | @staticmethod |
| 382 | def ProcessSummary( |
| 383 | df: pd.DataFrame, |
| 384 | start_time: Optional[float] = None, |
| 385 | end_time: Optional[float] = None, |
| 386 | pending_queries: int = 0 |
| 387 | ): |
| 388 | if start_time and end_time: |
| 389 | launched_queries = len( |
| 390 | df.query(f"{start_time} <= launch_time <= {end_time}") |
| 391 | ) |
| 392 | df = df.query(f"{start_time} <= finish_time <= {end_time}") |
| 393 | else: |
| 394 | launched_queries = len(df) |
| 395 | |
| 396 | logger.debug( |
| 397 | f"Launched queries: {launched_queries}, " |
| 398 | f"pending queries: {pending_queries}, " |
| 399 | f"finished queries: {len(df)}" |
| 400 | ) |
| 401 | |
| 402 | if start_time is None: |
| 403 | start_time = df["launch_time"].min() |
| 404 | if end_time is None: |
| 405 | end_time = df["finish_time"].max() |
| 406 | total_time = end_time - start_time |
| 407 | |
| 408 | total_requests = launched_queries + pending_queries |
| 409 | _qps = total_requests / total_time |
| 410 | |
| 411 | total_finished_requests = len(df) |
| 412 | finished_qps = total_finished_requests / total_time |
| 413 | |
| 414 | total_prompt_tokens = df["prompt_tokens"].sum() |
| 415 | total_generation_tokens = df["generation_tokens"].sum() |
| 416 | average_prefill_speed = total_prompt_tokens / total_time |
| 417 | average_generation_speed = total_generation_tokens / total_time |
| 418 | average_generation_speed_per_request = ( |
| 419 | df["generation_tokens"] / df["generation_time"] |
| 420 | ).mean() |
| 421 | average_ttft = df["ttft"].mean() |
| 422 | logger.info("Calculating performance summary") |
| 423 | print("\n") |
| 424 | print("==================== Performance summary ======================") |
| 425 | print( |
| 426 | f" \033[33mProcessing speed: " |
| 427 | f"\033[32m{finished_qps:.4f} reqs/s\033[0m\n" |
| 428 | ) |
| 429 | |
| 430 | print(f" \033[33mRequests on-the-fly: {pending_queries}\033[0m\n") |
| 431 | |
| 432 | print( |
| 433 | " \033[33mInput tokens per second: " |
| 434 | f"\033[32m{average_prefill_speed:.4f} tokens/s\033[0m\n" |
| 435 | ) |
| 436 | |
| 437 | print( |
| 438 | " \033[33mOutput tokens per second: " |
| 439 | f"\033[32m{average_generation_speed:.4f} tokens/s\033[0m\n" |