(args)
| 1059 | |
| 1060 | |
| 1061 | def run_benchmark(args): |
| 1062 | |
| 1063 | data_file = Path(args.data_dir) |
| 1064 | results_dir = Path(args.results_dir) / args.generation_model_name |
| 1065 | results_dir.mkdir(parents=True, exist_ok=True) |
| 1066 | |
| 1067 | print(f"🚀 Starting PlotCraft Benchmark Evaluation") |
| 1068 | print(f"Generation Model: {args.generation_model_name}") |
| 1069 | print(f"Evaluation Model: {args.evaluation_model_name}") |
| 1070 | print(f"Data Directory: {data_file}") |
| 1071 | print(f"Results Directory: {results_dir}") |
| 1072 | print("=" * 60) |
| 1073 | |
| 1074 | |
| 1075 | generation_cache_file = results_dir / "generation_cache.jsonl" |
| 1076 | generation_client = FluxOpenAIChat( |
| 1077 | base_url=args.generation_base_url, |
| 1078 | api_key=args.generation_api_key, |
| 1079 | cache_file=str(generation_cache_file), |
| 1080 | max_retries=args.max_retries, |
| 1081 | max_qps=args.max_qps, |
| 1082 | max_qpm=args.max_qpm, |
| 1083 | ) |
| 1084 | |
| 1085 | evaluation_cache_file = results_dir / "evaluation_cache.jsonl" |
| 1086 | evaluation_client = FluxOpenAIChat( |
| 1087 | base_url=args.evaluation_base_url, |
| 1088 | api_key=args.evaluation_api_key, |
| 1089 | cache_file=str(evaluation_cache_file), |
| 1090 | max_retries=args.max_retries, |
| 1091 | max_qps=args.max_qps, |
| 1092 | max_qpm=args.max_qpm, |
| 1093 | ) |
| 1094 | |
| 1095 | generation_prompt = read_system_prompt(args.generation_prompt_path) |
| 1096 | evaluation_prompt = read_system_prompt(args.evaluation_prompt_path) |
| 1097 | |
| 1098 | print("🔄 Preparing task information...") |
| 1099 | print("🔄 Loading task information...") |
| 1100 | task_infos = [] |
| 1101 | |
| 1102 | try: |
| 1103 | with open(data_file, 'r', encoding='utf-8') as f: |
| 1104 | data = json.load(f) |
| 1105 | |
| 1106 | # 直接读取JSON中的任务列表并转换路径 |
| 1107 | for task_key, task_info in data.items(): |
| 1108 | # 先修复路径字符串 |
| 1109 | if 'task_file' in task_info and task_info['task_file']: |
| 1110 | task_info['task_file'] = fix_path(task_info['task_file']) |
| 1111 | if 'data_directory' in task_info and task_info['data_directory']: |
| 1112 | task_info['data_directory'] = fix_path(task_info['data_directory']) |
| 1113 | if 'image_path' in task_info and task_info['image_path']: |
| 1114 | task_info['image_path'] = fix_path(task_info['image_path']) |
| 1115 | |
| 1116 | # 修复data_files字典中的路径 |
| 1117 | if 'data_files' in task_info and isinstance(task_info['data_files'], dict): |
| 1118 | fixed_data_files = {} |
no test coverage detected