| 722 | } |
| 723 | |
| 724 | async def _evaluate_batch( |
| 725 | self, |
| 726 | input_list: List[dict], |
| 727 | max_concurrent: int = 1, |
| 728 | output_path: Optional[str] = None # 新增参数 |
| 729 | ) -> List[dict]: |
| 730 | results = [] |
| 731 | |
| 732 | # 创建信号量控制并发数 |
| 733 | semaphore = asyncio.Semaphore(max_concurrent) |
| 734 | results = [None] * len(input_list) |
| 735 | |
| 736 | # 创建进度条和锁 |
| 737 | progress_bar = tqdm( |
| 738 | total=len(input_list), |
| 739 | desc="Evaling...", |
| 740 | colour="cyan", |
| 741 | dynamic_ncols=True, # 允许动态调整宽度 |
| 742 | unit_scale=False |
| 743 | ) |
| 744 | progress_lock = asyncio.Lock() |
| 745 | file_write_lock = asyncio.Lock() |
| 746 | |
| 747 | async def worker(idx, input_data): |
| 748 | async with semaphore: |
| 749 | result = await self._evaluate_one(input_data) |
| 750 | results[idx] = result |
| 751 | if output_path: |
| 752 | async with file_write_lock: |
| 753 | with open(output_path, "a", encoding="utf-8") as f: |
| 754 | try: |
| 755 | f.write(json.dumps(result, ensure_ascii=False) + "\n") |
| 756 | except Exception as e: |
| 757 | print(f"❌ 写入结果失败: {e}") |
| 758 | print(f"❌ 写入结果: {result}") |
| 759 | |
| 760 | # 任务完成时立即更新进度条 |
| 761 | async with progress_lock: |
| 762 | progress_bar.update(1) |
| 763 | |
| 764 | # 创建所有任务 |
| 765 | tasks = [worker(idx, input_data) for idx, input_data in enumerate(input_list)] |
| 766 | |
| 767 | # 等待所有任务完成 |
| 768 | await asyncio.gather(*tasks) |
| 769 | |
| 770 | # 关闭进度条 |
| 771 | progress_bar.close() |
| 772 | |
| 773 | return results |
| 774 | |
| 775 | def _load_bootcamp_registry(self, bootcamp_registry: str): |
| 776 | with jsonlines.open(bootcamp_registry) as reader: |