Function eval_from_file

scripts/eval_step.py:261–280

(args: argparse.Namespace, dataset, use_caption=True)

@beartype
def eval_from_file(args: argparse.Namespace, dataset, use_caption=True) -> None:
    result_path = Path(args.result_dir) / f"actions_{'caption' if use_caption else 'no_caption'}.json"
    with open(result_path, "r") as f:
        all_actions = json.load(f)
    all_labels = {}
    for example in dataset:
        all_labels[example["id"]] = example["label"]

    evaluator = StepEvaluator()

    correct = {}
    for example_id, action in all_actions.items():
        label = all_labels[example_id]
        correct[example_id] = evaluator(action, label)

    print(f"Benign success rate: {sum(correct.values())}/{len(all_actions)}")

    # write to file
    with open(Path(args.result_dir) / f"correct_{'caption' if use_caption else 'no_caption'}.json", "w") as f:
        json.dump(correct, f, indent=4)


def prepare(args: argparse.Namespace) -> None: