()
| 294 | |
| 295 | |
| 296 | def main(): |
| 297 | parser = argparse.ArgumentParser(description="Run SKU evaluation over a question JSON and generated videos (parallel).") |
| 298 | parser.add_argument("--concept_workers", type=int, default=2, help="Parallel workers across concepts.") |
| 299 | parser.add_argument("--per_question_workers", type=int, default=5, help="Parallel workers per concept per stage.") |
| 300 | parser.add_argument( |
| 301 | "--questions_json", |
| 302 | type=str, |
| 303 | default="/mlx_devbox/users/chenanno/playground/Code4Video/pipeline/json_files/questions_by_topic_10.json", |
| 304 | help="Path to the questions JSON file.", |
| 305 | ) |
| 306 | parser.add_argument( |
| 307 | "--concepts", |
| 308 | type=str, |
| 309 | nargs="*", |
| 310 | default=None, |
| 311 | help="Optional subset of concepts to evaluate. If not set, evaluate all in JSON.", |
| 312 | ) |
| 313 | # TODO: CASES 下的路径 |
| 314 | parser.add_argument( |
| 315 | "--base_dir", |
| 316 | type=str, |
| 317 | default="/mlx_devbox/users/chenanno/playground/Code4Video/pipeline/CASES/Sep_Gemini", |
| 318 | help="Base directory where per-knowledge-point video folders are located", |
| 319 | ) |
| 320 | # TODO: Test the number of knowledge points. If None, test all of them |
| 321 | parser.add_argument("--max_concepts", default=None) |
| 322 | args = parser.parse_args() |
| 323 | # 1) Load the question set |
| 324 | concept_questions = load_questions_from_json(args.questions_json) |
| 325 | all_concepts = list(concept_questions.keys()) |
| 326 | chosen_concepts = [c for c in all_concepts if (not args.concepts or c in args.concepts)] |
| 327 | if args.max_concepts is not None: |
| 328 | chosen_concepts = chosen_concepts[: args.max_concepts] |
| 329 | if not chosen_concepts: |
| 330 | print("[ERROR] No concepts to evaluate. Check --concepts or the JSON content.") |
| 331 | return |
| 332 | # 2) Generate a list of video paths |
| 333 | video_items = eva_video_list(chosen_concepts, args.base_dir) |
| 334 | concept2video = {item["knowledge_point"]: item["path"] for item in video_items} |
| 335 | # 3) Parallel execution |
| 336 | results: List[EvaluationResult] = [] |
| 337 | with ThreadPoolExecutor(max_workers=max(1, args.concept_workers)) as pool: |
| 338 | futures = {} |
| 339 | for concept in chosen_concepts: |
| 340 | qs = concept_questions.get(concept, []) |
| 341 | if not qs: |
| 342 | print(f"[WARN] No questions for concept '{concept}', skip.") |
| 343 | continue |
| 344 | vpath = concept2video.get(concept) |
| 345 | if not vpath: |
| 346 | print(f"[WARN] No video path for concept '{concept}', skip.") |
| 347 | continue |
| 348 | if not Path(vpath).exists(): |
| 349 | print(f"[WARN] Video file not found: {vpath} (concept '{concept}'). API may fail.") |
| 350 | fut = pool.submit(run_one_concept, concept, qs, vpath, args.per_question_workers) |
| 351 | futures[fut] = concept |
| 352 | for fut in as_completed(futures): |
| 353 | concept = futures[fut] |
no test coverage detected