Main function to run the evaluation script. Parses command line arguments and orchestrates the evaluation process for text, video, and image content using specified AI models.
()
| 347 | |
| 348 | |
| 349 | def main(): |
| 350 | """ |
| 351 | Main function to run the evaluation script. |
| 352 | |
| 353 | Parses command line arguments and orchestrates the evaluation process |
| 354 | for text, video, and image content using specified AI models. |
| 355 | """ |
| 356 | parser = argparse.ArgumentParser(description='Automatic evaluation of theorem explanation videos with LLMs') |
| 357 | parser.add_argument('--model_text', type=str, |
| 358 | choices=ALLOWED_MODELS, |
| 359 | default='azure/gpt-4o', |
| 360 | help='Select the AI model to use for text evaluation') |
| 361 | parser.add_argument('--model_video', type=str, |
| 362 | choices=['gemini/gemini-1.5-pro-002', |
| 363 | 'gemini/gemini-2.0-flash-exp', |
| 364 | 'gemini/gemini-2.0-pro-exp-02-05'], |
| 365 | default='gemini/gemini-1.5-pro-002', |
| 366 | help='Select the AI model to use for video evaluation') |
| 367 | parser.add_argument('--model_image', type=str, |
| 368 | choices=ALLOWED_MODELS, |
| 369 | default='azure/gpt-4o', |
| 370 | help='Select the AI model to use for image evaluation') |
| 371 | parser.add_argument('--eval_type', type=str, choices=['text', 'video', 'image', 'all'], default='all', help='Type of evaluation to perform') |
| 372 | parser.add_argument('--file_path', type=str, help='Path to a file or a theorem folder', required=True) |
| 373 | parser.add_argument('--output_folder', type=str, help='Directory to store the evaluation files', required=True) |
| 374 | parser.add_argument('--retry_limit', type=int, default=3, help='Number of retry attempts for each inference') |
| 375 | parser.add_argument('--combine', action='store_true', help='Combine all results into a single JSON file') |
| 376 | parser.add_argument('--bulk_evaluate', action='store_true', help='Evaluate a folder of theorems together', default=False) |
| 377 | parser.add_argument('--target_fps', type=int, help='Target FPS for video processing. If not set, original video FPS will be used', required=False) |
| 378 | parser.add_argument('--use_parent_folder_as_topic', action='store_true', help='Use parent folder name as topic name for single file evaluation', default=True) |
| 379 | parser.add_argument('--max_workers', type=int, default=4, help='Maximum number of concurrent workers for parallel processing') |
| 380 | |
| 381 | args = parser.parse_args() |
| 382 | |
| 383 | # Initialize separate models |
| 384 | text_model = LiteLLMWrapper( |
| 385 | model_name=args.model_text, |
| 386 | temperature=0.0, |
| 387 | ) |
| 388 | video_model = GeminiWrapper( |
| 389 | model_name=args.model_video, |
| 390 | temperature=0.0, |
| 391 | ) |
| 392 | image_model = LiteLLMWrapper( |
| 393 | model_name=args.model_image, |
| 394 | temperature=0.0, |
| 395 | ) |
| 396 | |
| 397 | models = { |
| 398 | 'text': text_model, |
| 399 | 'video': video_model, |
| 400 | 'image': image_model |
| 401 | } |
| 402 | |
| 403 | theorem_dirs = [] |
| 404 | if args.bulk_evaluate: |
| 405 | assert os.path.isdir(args.file_path), "File path must be a folder for --bulk_evaluate" |
| 406 | for root, dirnames, _ in os.walk(args.file_path): |
no test coverage detected