(args_in: list[str] | None = None)
| 22 | |
| 23 | |
| 24 | def main(args_in: list[str] | None = None) -> None: |
| 25 | parser = argparse.ArgumentParser(description="Start server benchmark scenario") |
| 26 | parser.add_argument("--name", type=str, help="Bench name", required=True) |
| 27 | parser.add_argument("--runner-label", type=str, help="Runner label", required=True) |
| 28 | parser.add_argument("--branch", type=str, help="Branch name", default="detached") |
| 29 | parser.add_argument("--commit", type=str, help="Commit name", default="dirty") |
| 30 | parser.add_argument("--host", type=str, help="Server listen host", default="0.0.0.0") |
| 31 | parser.add_argument("--port", type=int, help="Server listen host", default="8080") |
| 32 | parser.add_argument("--model-path-prefix", type=str, help="Prefix where to store the model files", default="models") |
| 33 | parser.add_argument("--n-prompts", type=int, |
| 34 | help="SERVER_BENCH_N_PROMPTS: total prompts to randomly select in the benchmark", required=True) |
| 35 | parser.add_argument("--max-prompt-tokens", type=int, |
| 36 | help="SERVER_BENCH_MAX_PROMPT_TOKENS: maximum prompt tokens to filter out in the dataset", |
| 37 | required=True) |
| 38 | parser.add_argument("--max-tokens", type=int, |
| 39 | help="SERVER_BENCH_MAX_CONTEXT: maximum context size of the completions request to filter out in the dataset: prompt + predicted tokens", |
| 40 | required=True) |
| 41 | parser.add_argument("--hf-repo", type=str, help="Hugging Face model repository", required=True) |
| 42 | parser.add_argument("--hf-file", type=str, help="Hugging Face model file", required=True) |
| 43 | parser.add_argument("-ngl", "--n-gpu-layers", type=int, help="layers to the GPU for computation", required=True) |
| 44 | parser.add_argument("--ctx-size", type=int, help="Set the size of the prompt context", required=True) |
| 45 | parser.add_argument("--parallel", type=int, help="Set the number of slots for process requests", required=True) |
| 46 | parser.add_argument("--batch-size", type=int, help="Set the batch size for prompt processing", required=True) |
| 47 | parser.add_argument("--ubatch-size", type=int, help="physical maximum batch size", required=True) |
| 48 | parser.add_argument("--scenario", type=str, help="Scenario to run", required=True) |
| 49 | parser.add_argument("--duration", type=str, help="Bench scenario", required=True) |
| 50 | |
| 51 | args = parser.parse_args(args_in) |
| 52 | |
| 53 | start_time = time.time() |
| 54 | |
| 55 | # Start the server and performance scenario |
| 56 | try: |
| 57 | server_process = start_server(args) |
| 58 | except Exception: |
| 59 | print("bench: server start error :") |
| 60 | traceback.print_exc(file=sys.stdout) |
| 61 | sys.exit(1) |
| 62 | |
| 63 | # start the benchmark |
| 64 | iterations = 0 |
| 65 | data = {} |
| 66 | try: |
| 67 | start_benchmark(args) |
| 68 | |
| 69 | with open("results.github.env", 'w') as github_env: |
| 70 | # parse output |
| 71 | with open('k6-results.json', 'r') as bench_results: |
| 72 | # Load JSON data from file |
| 73 | data = json.load(bench_results) |
| 74 | for metric_name in data['metrics']: |
| 75 | for metric_metric in data['metrics'][metric_name]: |
| 76 | value = data['metrics'][metric_name][metric_metric] |
| 77 | if isinstance(value, float) or isinstance(value, int): |
| 78 | value = round(value, 2) |
| 79 | data['metrics'][metric_name][metric_metric]=value |
| 80 | github_env.write( |
| 81 | f"{escape_metric_name(metric_name)}_{escape_metric_name(metric_metric)}={value}\n") |
no test coverage detected