(args)
| 1067 | |
| 1068 | |
| 1069 | def run_benchmark(args): |
| 1070 | |
| 1071 | data_file = Path(args.data_dir) |
| 1072 | results_dir = Path(args.results_dir) / args.generation_model_name |
| 1073 | results_dir.mkdir(parents=True, exist_ok=True) |
| 1074 | |
| 1075 | print(f"🚀 Starting PlotCraft Benchmark Evaluation") |
| 1076 | print(f"Generation Model: {args.generation_model_name}") |
| 1077 | print(f"Evaluation Model: {args.evaluation_model_name}") |
| 1078 | print(f"Data Directory: {data_file}") |
| 1079 | print(f"Results Directory: {results_dir}") |
| 1080 | print("=" * 60) |
| 1081 | |
| 1082 | |
| 1083 | generation_cache_file = results_dir / "generation_cache.jsonl" |
| 1084 | generation_client = FluxOpenAIChat( |
| 1085 | base_url=args.generation_base_url, |
| 1086 | api_key=args.generation_api_key, |
| 1087 | cache_file=str(generation_cache_file), |
| 1088 | max_retries=args.max_retries, |
| 1089 | max_qps=args.max_qps, |
| 1090 | max_qpm=args.max_qpm, |
| 1091 | ) |
| 1092 | |
| 1093 | evaluation_cache_file = results_dir / "evaluation_cache.jsonl" |
| 1094 | evaluation_client = FluxOpenAIChat( |
| 1095 | base_url=args.evaluation_base_url, |
| 1096 | api_key=args.evaluation_api_key, |
| 1097 | cache_file=str(evaluation_cache_file), |
| 1098 | max_retries=args.max_retries, |
| 1099 | max_qps=args.max_qps, |
| 1100 | max_qpm=args.max_qpm, |
| 1101 | ) |
| 1102 | |
| 1103 | generation_prompt = read_system_prompt(args.generation_prompt_path) |
| 1104 | user_history_generation_prompt = read_system_prompt(args.user_history_generation_prompt_path) |
| 1105 | evaluation_prompt = read_system_prompt(args.evaluation_prompt_path) |
| 1106 | |
| 1107 | print("🔄 Preparing task information...") |
| 1108 | print("🔄 Loading task information...") |
| 1109 | task_infos = [] |
| 1110 | |
| 1111 | try: |
| 1112 | with open(data_file, 'r', encoding='utf-8') as f: |
| 1113 | data = json.load(f) |
| 1114 | |
| 1115 | # 直接读取JSON中的任务列表并转换路径 |
| 1116 | for task_key, task_info in data.items(): |
| 1117 | # 转换路径字段为Path对象 |
| 1118 | if 'task_file' in task_info and task_info['task_file']: |
| 1119 | task_info['task_file'] = Path(task_info['task_file']) |
| 1120 | if 'data_directory' in task_info and task_info['data_directory']: |
| 1121 | task_info['data_directory'] = Path(task_info['data_directory']) |
| 1122 | if 'image_path' in task_info and task_info['image_path']: |
| 1123 | task_info['image_path'] = Path(task_info['image_path']) |
| 1124 | if 'refine_before_image' in task_info and task_info['refine_before_image']: |
| 1125 | task_info['refine_before_image'] = Path(task_info['refine_before_image']) |
| 1126 |
no test coverage detected