(args)
| 138 | raise ExecError(f"An error occurred, indicating unhandled edge case.") |
| 139 | |
| 140 | def evaluate(args): |
| 141 | |
| 142 | # Get dataset path |
| 143 | args.data_dir, args.split, _ = get_dataset_dir(args.data_name) |
| 144 | |
| 145 | # Convert to aitz format |
| 146 | convert2aitz(os.path.abspath(args.input_path), os.path.abspath(args.output_dir), max_workers=16) |
| 147 | |
| 148 | save_dir = os.path.abspath(args.output_dir) |
| 149 | results_save_file = os.path.join(save_dir, "result.json") |
| 150 | |
| 151 | # Initialize dataset |
| 152 | eval_data = EvalDataset(data_dir=args.data_dir) |
| 153 | logging.info(f"Total steps: {len(eval_data)}, total episodes: {len(eval_data.episode_data)}.") |
| 154 | |
| 155 | evaluator = ActionEvaluator(save_dir, args.eval_android_control) |
| 156 | results = list(tqdm(map(process_step_data, eval_data.data, [evaluator]*len(eval_data.data), [save_dir]*len(eval_data.data)),total=len(eval_data.data), desc="Processing steps", ncols=100)) |
| 157 | results = list(filter(lambda x: x is not None, results)) |
| 158 | |
| 159 | # Save results |
| 160 | try: |
| 161 | os.makedirs(os.path.dirname(results_save_file), exist_ok=True) |
| 162 | with open(results_save_file, "w") as f: |
| 163 | json.dump(results, f, indent=4, ensure_ascii=False) |
| 164 | logging.info(f"Evaluation results saved to {results_save_file}") |
| 165 | except Exception as e: |
| 166 | logging.error(f"Error saving results to {results_save_file}: {e}") |
| 167 | |
| 168 | # Aggregate episode results |
| 169 | episode_results = defaultdict(list) |
| 170 | for result in results: |
| 171 | subset = result.get("subset") |
| 172 | episode_id = result.get("episode_id") |
| 173 | if subset is None or episode_id is None: |
| 174 | logging.warning(f"Result missing subset/episode_id: {result}") |
| 175 | continue |
| 176 | episode_key = f"{subset}-{episode_id}" |
| 177 | episode_results[episode_key].append(result) |
| 178 | |
| 179 | # Compute final evaluation metrics |
| 180 | try: |
| 181 | episode_metrics = ActionEvaluator.compute_episode_metrics(episode_results) |
| 182 | atomic_metrics = ActionEvaluator.compute_atomic_metrics(results) |
| 183 | logging.basicConfig(level=logging.INFO, format='%(message)s') |
| 184 | logging.info(f"episode_metrics: {episode_metrics}") |
| 185 | logging.info(f"atomic_metrics: {atomic_metrics}") |
| 186 | logging.info( |
| 187 | f"success_rate: {episode_metrics.get('success_rate')}, " |
| 188 | f"goal_progress: {episode_metrics.get('goal_progress')}, " |
| 189 | f"type_acc: {atomic_metrics.get('total', {}).get('type_acc')}, " |
| 190 | f"exact_acc: {atomic_metrics.get('total', {}).get('exact_acc')}" |
| 191 | ) |
| 192 | |
| 193 | summary_save_file = os.path.join(args.output_dir, 'summary.json') |
| 194 | logging.info(f"Evaluation summary saved to {summary_save_file}") |
| 195 | with open(summary_save_file, 'w') as f: |
| 196 | json.dump(episode_metrics|atomic_metrics, f, ensure_ascii=False) |
| 197 |
no test coverage detected