Run the full evaluation pipeline. Args: data_path: Directory containing generated images. csv_file: CSV file with prompts and checkpoints. lang: Language code ('en' or 'zh'). backend: Backend type ('gemini' or 'vllm').
(
data_path,
csv_file,
lang,
backend,
api_key=None,
base_url=None,
api_url=None,
model_name=None,
num_processes=20,
images_per_prompt=4,
image_suffix=".png",
max_retries=10,
resume=False,
category=None,
)
| 479 | # ============================================================ |
| 480 | |
| 481 | def run_evaluation( |
| 482 | data_path, |
| 483 | csv_file, |
| 484 | lang, |
| 485 | backend, |
| 486 | api_key=None, |
| 487 | base_url=None, |
| 488 | api_url=None, |
| 489 | model_name=None, |
| 490 | num_processes=20, |
| 491 | images_per_prompt=4, |
| 492 | image_suffix=".png", |
| 493 | max_retries=10, |
| 494 | resume=False, |
| 495 | category=None, |
| 496 | ): |
| 497 | """Run the full evaluation pipeline. |
| 498 | |
| 499 | Args: |
| 500 | data_path: Directory containing generated images. |
| 501 | csv_file: CSV file with prompts and checkpoints. |
| 502 | lang: Language code ('en' or 'zh'). |
| 503 | backend: Backend type ('gemini' or 'vllm'). |
| 504 | api_key: API key (gemini backend). |
| 505 | base_url: Base URL for API (gemini backend). |
| 506 | api_url: vLLM server URL (vllm backend). |
| 507 | model_name: Model name for API calls (gemini backend). |
| 508 | num_processes: Number of parallel worker processes. |
| 509 | images_per_prompt: Number of generated images per prompt. |
| 510 | image_suffix: Image file extension (e.g. '.png'). |
| 511 | max_retries: Maximum retry attempts per evaluation. |
| 512 | resume: Resume from previous progress file. |
| 513 | category: Category label (e.g. 'en', 'en_long', 'zh', 'zh_long') |
| 514 | used for output file naming. Falls back to lang if not set. |
| 515 | """ |
| 516 | if lang not in LANG_CONFIGS: |
| 517 | raise ValueError(f"Unsupported language '{lang}'. Choose from: {list(LANG_CONFIGS.keys())}") |
| 518 | |
| 519 | lang_config = LANG_CONFIGS[lang] |
| 520 | file_name = os.path.basename(os.path.normpath(data_path)) |
| 521 | tag = category or lang |
| 522 | out_file = f"./results/{file_name}_{tag}.csv" |
| 523 | progress_file = f"{out_file}.progress.jsonl" |
| 524 | |
| 525 | os.makedirs(os.path.dirname(out_file), exist_ok=True) |
| 526 | |
| 527 | # Handle resume vs fresh start |
| 528 | completed = {} |
| 529 | if resume and os.path.exists(progress_file): |
| 530 | completed = load_completed_results(progress_file) |
| 531 | print(f"Resuming: loaded {len(completed)} completed results") |
| 532 | else: |
| 533 | for fpath in [out_file, progress_file]: |
| 534 | if os.path.exists(fpath): |
| 535 | os.remove(fpath) |
| 536 | print(f"Removed existing file: {fpath}") |
| 537 | |
| 538 | # Read CSV and prepare tasks |
no test coverage detected