Function eval_experiment

utils/src/experiment/evals.py:198–232

(
    setting: str,
    general_eval: bool = False,
    feature_eval: bool = False,
    ppt_eval: bool = False,
)

# ppt eval
def eval_experiment(
    setting: str,
    general_eval: bool = False,
    feature_eval: bool = False,
    ppt_eval: bool = False,
):
    assert setting != "*"
    llms.language_model, llms.vision_model, judge_name = judges[0]
    print(f"evaluating {setting} under {judge_name}")
    print(
        "eval config :",
        f"general_eval: {general_eval}, feature_eval: {feature_eval}, ppt_eval: {ppt_eval}",
    )
    eval_file = f"data/evals/{setting}_{judge_name}.json"
    eval_stats = defaultdict(dict)
    if pexists(eval_file):
        eval_stats |= json.load(open(eval_file))
    config = Config("/tmp")
    prs_files = glob(f"data/*/pptx/*/{setting}/*/final.pptx")
    # filename dimension score
    print("start evaluation")
    if general_eval or feature_eval:
        presentations = [Presentation.from_file(i, config) for i in prs_files]
    if general_eval:
        eval_general(presentations, eval_stats)

    if feature_eval:
        eval_feature(presentations, eval_stats, setting)

    if ppt_eval:
        slide_image_folders = glob(f"data/*/pptx/*/final_images/{setting}/*")
        for presentation in prs_files:
            pres_score(presentation)
        eval_stats = merge_evals(slide_image_folders, eval_stats)
    json.dump(eval_stats, open(eval_file, "w"), indent=4)


def eval_baseline(