Function eval_baseline

utils/src/experiment/evals.py:235–261

(
    setting: str,
    model: Literal["Qwen2.5", "gpt-4o"],
    general_eval: bool = False,
    feature_eval: bool = False,
    ppt_eval: bool = False,
)

def eval_baseline(
    setting: str,
    model: Literal["Qwen2.5", "gpt-4o"],
    general_eval: bool = False,
    feature_eval: bool = False,
    ppt_eval: bool = False,
):
    evals = defaultdict(dict)
    prs_files = glob(f"data/*/pdf/*/{setting}/{model}/final.pptx")
    slide_folders = [os.path.dirname(i) for i in prs_files]

    if general_eval or feature_eval:
        config = Config("/tmp")
        presentations = [Presentation.from_file(i, config) for i in prs_files]

    if general_eval:
        eval_general(presentations, evals)
    if feature_eval:
        eval_feature(presentations, evals, setting, fid_eval=False)
    if ppt_eval:
        for slide_folder in slide_folders:
            slide_score(slide_folder)
        for presentation in prs_files:
            pres_score(presentation)

    merge_evals(slide_folders, evals)
    json.dump(evals, open(f"data/evals/{setting}_{model}.json", "w"), indent=4)


if __name__ == "__main__":