(prs_source: str)
| 156 | |
| 157 | |
| 158 | def pres_score(prs_source: str): |
| 159 | if "/pptx/" in prs_source: # ours |
| 160 | source, setting, pdf, _ = prs_source.rsplit("/", 3) |
| 161 | slide_folder = os.path.join(source, "final_images", setting, pdf) |
| 162 | else: # baseline |
| 163 | slide_folder = os.path.dirname(prs_source) |
| 164 | eval_file = pjoin(slide_folder, "evals.json") |
| 165 | evals = defaultdict(dict) |
| 166 | if pexists(eval_file): |
| 167 | try: |
| 168 | evals |= json.load(open(eval_file)) |
| 169 | except: |
| 170 | pass |
| 171 | evals.pop("logic", None) # ? debug |
| 172 | |
| 173 | slide_descr = pjoin(slide_folder, "extracted.json") |
| 174 | if not pexists(slide_descr): |
| 175 | config = Config("/tmp") |
| 176 | presentation = Presentation.from_file(prs_source, config) |
| 177 | ppt_extractor = Template(open("prompts/ppteval_extract.txt", "r").read()) |
| 178 | extracted = llms.language_model( |
| 179 | ppt_extractor.render(presentation=presentation.to_text()), |
| 180 | return_json=True, |
| 181 | ) |
| 182 | json.dump(extracted, open(slide_descr, "w"), indent=4) |
| 183 | else: |
| 184 | extracted = json.load(open(slide_descr)) |
| 185 | if presentation.source_file not in evals["logic"]: |
| 186 | logic_scorer = Template(open("ppteval_coherence.txt", "r").read()) |
| 187 | evals["logic"][presentation.source_file] = llms.language_model( |
| 188 | logic_scorer.render( |
| 189 | background_information=extracted.pop("metadata"), |
| 190 | logical_structure=extracted, |
| 191 | ), |
| 192 | return_json=True, |
| 193 | ) |
| 194 | json.dump(evals, open(eval_file, "w"), indent=4) |
| 195 | |
| 196 | |
| 197 | # ppt eval |
no test coverage detected