(args: argparse.Namespace, dataset, use_caption=True)
| 322 | |
| 323 | @beartype |
| 324 | def eval_from_file(args: argparse.Namespace, dataset, use_caption=True) -> None: |
| 325 | result_path = Path(args.result_dir) / f"actions_{args.attack}_{'' if use_caption else 'no_'}cap.json" |
| 326 | with open(result_path, "r") as f: |
| 327 | all_actions = json.load(f) |
| 328 | all_target_labels = {} |
| 329 | for example in dataset: |
| 330 | all_target_labels[example["id"]] = example["target_label"] |
| 331 | |
| 332 | evaluator = StepEvaluator() |
| 333 | |
| 334 | target_correct = {} |
| 335 | for example_id, action in all_actions.items(): |
| 336 | target_label = all_target_labels[example_id] |
| 337 | target_correct[example_id] = evaluator(action, target_label) |
| 338 | |
| 339 | print(f"Attack success rate: {sum(target_correct.values())}/{len(all_actions)}") |
| 340 | |
| 341 | # write to file |
| 342 | with open( |
| 343 | Path(args.result_dir) / f"target_correct_{args.attack}_{'' if use_caption else 'no_'}cap.json", |
| 344 | "w", |
| 345 | ) as f: |
| 346 | json.dump(target_correct, f, indent=4) |
| 347 | |
| 348 | |
| 349 | def prepare(args: argparse.Namespace) -> None: |
no test coverage detected