| 1024 | |
| 1025 | |
| 1026 | async def main_async(args: argparse.Namespace): |
| 1027 | print(args) |
| 1028 | random.seed(args.seed) |
| 1029 | np.random.seed(args.seed) |
| 1030 | |
| 1031 | # Validate ramp-up arguments |
| 1032 | if args.ramp_up_strategy is not None: |
| 1033 | if args.request_rate != float("inf"): |
| 1034 | raise ValueError( |
| 1035 | "When using ramp-up, do not specify --request-rate. " |
| 1036 | "The request rate will be controlled by ramp-up parameters. " |
| 1037 | "Please remove the --request-rate argument." |
| 1038 | ) |
| 1039 | if args.ramp_up_start_rps is None or args.ramp_up_end_rps is None: |
| 1040 | raise ValueError( |
| 1041 | "When using --ramp-up-strategy, both --ramp-up-start-rps and " "--ramp-up-end-rps must be specified" |
| 1042 | ) |
| 1043 | if args.ramp_up_start_rps < 0 or args.ramp_up_end_rps < 0: |
| 1044 | raise ValueError("Ramp-up start and end RPS must be non-negative") |
| 1045 | if args.ramp_up_start_rps > args.ramp_up_end_rps: |
| 1046 | raise ValueError("Ramp-up start RPS must be less than end RPS") |
| 1047 | if args.ramp_up_strategy == "exponential" and args.ramp_up_start_rps == 0: |
| 1048 | raise ValueError("For exponential ramp-up, the start RPS cannot be 0.") |
| 1049 | |
| 1050 | endpoint_type = args.backend |
| 1051 | backend = args.backend |
| 1052 | label = args.label |
| 1053 | model_id = args.model |
| 1054 | model_name = args.served_model_name |
| 1055 | tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model |
| 1056 | |
| 1057 | if args.base_url is not None: |
| 1058 | api_url = f"{args.base_url}{args.endpoint}" |
| 1059 | base_url = f"{args.base_url}" |
| 1060 | else: |
| 1061 | api_url = f"http://{args.host}:{args.port}{args.endpoint}" |
| 1062 | base_url = f"http://{args.host}:{args.port}" |
| 1063 | print(f"API URL: {api_url}") |
| 1064 | print(f"base URL: {base_url}") |
| 1065 | |
| 1066 | # Headers |
| 1067 | headers = None |
| 1068 | if args.header: |
| 1069 | headers = {} |
| 1070 | for item in args.header: |
| 1071 | if "=" in item: |
| 1072 | kvstring = item.split("=", 1) |
| 1073 | headers[kvstring[0].strip()] = kvstring[1].strip() |
| 1074 | else: |
| 1075 | raise ValueError("Invalid header format. Please use KEY=VALUE format.") |
| 1076 | |
| 1077 | if args.dataset_name is None: |
| 1078 | raise ValueError("Please specify '--dataset-name' and the corresponding " "'--dataset-path' if required.") |
| 1079 | |
| 1080 | # Load the dataset. |
| 1081 | input_requests = get_samples(args) |
| 1082 | goodput_config_dict = check_goodput_args(args) |
| 1083 | |