| 32 | |
| 33 | |
| 34 | def get_ppl(slide: SlidePage, model: GPT2LMHeadModel, tokenizer: GPT2TokenizerFast): |
| 35 | ppl = [] |
| 36 | text = slide.to_text() |
| 37 | if len(text) == 0: |
| 38 | return ppl |
| 39 | tokenized = tokenizer(text, return_tensors="pt").to(model.device) |
| 40 | with torch.no_grad(): |
| 41 | outputs = model(tokenized.input_ids, labels=tokenized.input_ids) |
| 42 | loss = outputs.loss |
| 43 | perplexity = torch.exp(loss) |
| 44 | ppl.append(perplexity.item()) |
| 45 | return ppl |
| 46 | |
| 47 | |
| 48 | def eval_general(presentations: list[Presentation], evals: dict[str, list[int]]): |