()
| 55 | |
| 56 | |
| 57 | def main(): |
| 58 | args = parse_args() |
| 59 | |
| 60 | print(f'MBench Evaluation Arguments: {args}') |
| 61 | |
| 62 | # Initialize MBench |
| 63 | device = torch.device(args.device) |
| 64 | mbench = MBench( |
| 65 | device=str(device), |
| 66 | output_path=args.output_path, |
| 67 | full_info_dir=args.full_info_json |
| 68 | ) |
| 69 | |
| 70 | # Prepare kwargs for evaluation |
| 71 | kwargs = {} |
| 72 | if args.gemini_api_key: |
| 73 | kwargs['api_key'] = args.gemini_api_key |
| 74 | |
| 75 | # Generate evaluation name with timestamp |
| 76 | current_time = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') |
| 77 | eval_name = f'mbench_results_{current_time}' |
| 78 | |
| 79 | print(f'Starting MBench evaluation...') |
| 80 | print(f'Evaluation path: {args.evaluation_path}') |
| 81 | print(f'Dimensions: {args.dimension}') |
| 82 | |
| 83 | # Run evaluation |
| 84 | try: |
| 85 | results = mbench.evaluate( |
| 86 | evaluation_path=args.evaluation_path, |
| 87 | name=eval_name, |
| 88 | dimension_list=args.dimension, |
| 89 | **kwargs |
| 90 | ) |
| 91 | |
| 92 | # Print summary |
| 93 | print('\nEvaluation completed successfully!') |
| 94 | print(f'Results saved to: {args.output_path}') |
| 95 | |
| 96 | # Print brief results summary |
| 97 | print('\nResults Summary:') |
| 98 | print('-' * 40) |
| 99 | for dimension, result in results.items(): |
| 100 | if isinstance(result, dict) and 'error' not in result: |
| 101 | if 'aggregate' in result: |
| 102 | agg = result['aggregate'] |
| 103 | print(f'{dimension}: {agg.get("mean", "N/A"):.4f} (std: {agg.get("std", 0):.4f})') |
| 104 | elif 'accuracy' in result: |
| 105 | print(f'{dimension}: {result["accuracy"]:.4f}') |
| 106 | elif 'score' in result: |
| 107 | print(f'{dimension}: {result["score"]:.4f}') |
| 108 | else: |
| 109 | print(f'{dimension}: Completed') |
| 110 | elif isinstance(result, dict) and 'error' in result: |
| 111 | print(f'{dimension}: Error - {result["error"]}') |
| 112 | else: |
| 113 | print(f'{dimension}: {result}') |
| 114 |
no test coverage detected