| 1532 | |
| 1533 | |
| 1534 | def measure_cold(loader: LoaderFn, case: dict[str, Any], request: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: |
| 1535 | wall_ms: list[float] = [] |
| 1536 | duration_sec = 0.0 |
| 1537 | for _ in range(3): |
| 1538 | started = time.perf_counter() |
| 1539 | _model, run = loader(case, args) |
| 1540 | run_wall_ms, duration_sec = run(request, args) |
| 1541 | total_ms = (time.perf_counter() - started) * 1000.0 |
| 1542 | if total_ms < run_wall_ms: |
| 1543 | total_ms = run_wall_ms |
| 1544 | wall_ms.append(total_ms) |
| 1545 | average_wall_ms = float(sum(wall_ms) / len(wall_ms)) |
| 1546 | return { |
| 1547 | "wall_ms": wall_ms, |
| 1548 | "average_wall_ms": average_wall_ms, |
| 1549 | "duration_sec": duration_sec, |
| 1550 | "rtf": (average_wall_ms / 1000.0 / duration_sec) if duration_sec > 0.0 else 0.0, |
| 1551 | } |
| 1552 | |
| 1553 | |
| 1554 | def measure_warm(loader: LoaderFn, case: dict[str, Any], warmup_request: dict[str, Any], request: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: |