(
presentations: list[Presentation],
evals: dict,
setting: str,
)
| 59 | |
| 60 | |
| 61 | def eval_feature( |
| 62 | presentations: list[Presentation], |
| 63 | evals: dict, |
| 64 | setting: str, |
| 65 | ): |
| 66 | device = f"cuda:{random.randint(0, DEVICES - 1)}" |
| 67 | print("start scoring ppl") |
| 68 | model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) |
| 69 | tokenizer = GPT2TokenizerFast.from_pretrained("gpt2") |
| 70 | for prs in tqdm(presentations): |
| 71 | try: |
| 72 | if prs.source_file in evals["ppl"]: |
| 73 | continue |
| 74 | if ( |
| 75 | prs.source_file |
| 76 | == "data/culture/pptx/ChemBio-in-the-HUB-public/PPTCrew_wo_SchemaInduction/SSRN-id2933553_Management of Systems Engineering and Technical Assistance of DARPA Research Programs/final.pptx" |
| 77 | ): |
| 78 | continue |
| 79 | ppl = [] |
| 80 | for slide in prs.slides: |
| 81 | ppl.extend(get_ppl(slide, model, tokenizer)) |
| 82 | if len(ppl) == 0: |
| 83 | continue |
| 84 | evals["ppl"][prs.source_file] = sum(ppl) / len(ppl) |
| 85 | except Exception as e: |
| 86 | print(e, "\n", "happended in ", prs.source_file) |
| 87 | |
| 88 | model = fid.InceptionV3([fid.InceptionV3.BLOCK_INDEX_BY_DIM[64]]).to(device) |
| 89 | for ppt_folder in tqdm(sorted(glob(f"data/*/pptx/*/"))): |
| 90 | if ppt_folder in evals["fid"]: |
| 91 | continue |
| 92 | source_folder = pjoin(ppt_folder, "source_slides") |
| 93 | m1, s1 = compute_statistics_of_path(source_folder, model, 128, 64, device) |
| 94 | try: |
| 95 | with tempfile.TemporaryDirectory(prefix="ppteval_fid_") as temp_dir: |
| 96 | for result_folder in glob( |
| 97 | pjoin(ppt_folder, f"final_images/{setting}/*") |
| 98 | ): |
| 99 | folder_base = os.path.basename(result_folder) |
| 100 | for image_file in os.listdir(result_folder): |
| 101 | image_path = os.path.join(result_folder, image_file) |
| 102 | temp_image_path = os.path.join( |
| 103 | temp_dir, folder_base + "_" + image_file |
| 104 | ).replace(" ", "_") |
| 105 | shutil.copyfile(image_path, temp_image_path) |
| 106 | if len(os.listdir(temp_dir)) < 10: |
| 107 | continue |
| 108 | m2, s2 = compute_statistics_of_path(temp_dir, model, 32, 64, device) |
| 109 | |
| 110 | evals["fid"][ppt_folder] = fid.calculate_frechet_distance( |
| 111 | m1, s1, m2, s2 |
| 112 | ) |
| 113 | except Exception as e: |
| 114 | print(e, "\n", "happended in ", ppt_folder, "on:", setting) |
| 115 | |
| 116 | |
| 117 | def merge_evals(folders: list[str], evals: dict): |
no test coverage detected