MCPcopy Create free account
hub / github.com/NatsUIJM/autoContents / run_batch_processing

Function run_batch_processing

mainprogress/determine_toc_levels.py:442–482  ·  view source on GitHub ↗
(image_files: list, output_path: Path)

Source from the content-addressed store, hash-verified

440 print("首图处理失败,脚本停止。请检查日志。")
441 return
442
443 # 2. 并发处理剩余图片
444 if len(image_files) > 1:
445 write_log("阶段 2: 基于首图示例并发处理剩余图片")
446 tasks = []
447 for img_file in image_files[1:]:
448 csv_file = output_path / f"{img_file.stem}.csv"
449 tasks.append(process_level_async(semaphore, img_file, csv_file, output_path))
450
451 results = await asyncio.gather(*tasks, return_exceptions=True)
452 success_count = sum(1 for r in results if r is True)
453 fail_count = sum(1 for r in results if r is False)
454 exception_count = sum(1 for r in results if isinstance(r, Exception))
455
456 print(f"并发处理完成。成功:{success_count}, 失败/空结果:{fail_count}, 异常:{exception_count}")
457 else:
458 print("仅有一张图片,处理完毕。")
459
460 # 处理完成后清空图片缓存,释放内存
461 IMAGE_CACHE.clear()
462
463async def main_async():
464 load_dotenv()
465
466 global LLM_CONFIG, client
467 try:
468 LLM_CONFIG = load_llm_config()
469 except Exception as e:
470 print(f"加载 LLM 配置失败:{e}")
471 sys.exit(1)
472
473 # 初始化 OpenAI 客户端
474 client = AsyncOpenAI(
475 api_key=LLM_CONFIG["api_key"],
476 base_url=LLM_CONFIG["base_url"],
477 timeout=REQUEST_TIMEOUT,
478 max_retries=2
479 )
480
481 base_dir = os.getenv("BASE_DIR")
482 if base_dir:
483 input_path = Path(base_dir) / "mark" / "input_image"
484 output_path = Path(base_dir) / "raw_content"
485 else:

Callers 1

main_asyncFunction · 0.70

Calls 4

process_first_pageFunction · 0.85
process_level_asyncFunction · 0.85
write_logFunction · 0.70
get_encoded_imageFunction · 0.70

Tested by

no test coverage detected