Deduplicate slides in a presentation based on text similarity. Args: presentation (Presentation): The presentation object containing slides. model: The model used for generating text embeddings. batchsize (int): The batch size for processing slides. threshol
(
presentation: Presentation,
model: BGEM3FlagModel,
batchsize: int = 32,
threshold: float = 0.8,
)
| 20 | |
| 21 | |
| 22 | def prs_dedup( |
| 23 | presentation: Presentation, |
| 24 | model: BGEM3FlagModel, |
| 25 | batchsize: int = 32, |
| 26 | threshold: float = 0.8, |
| 27 | ) -> list[SlidePage]: |
| 28 | """ |
| 29 | Deduplicate slides in a presentation based on text similarity. |
| 30 | |
| 31 | Args: |
| 32 | presentation (Presentation): The presentation object containing slides. |
| 33 | model: The model used for generating text embeddings. |
| 34 | batchsize (int): The batch size for processing slides. |
| 35 | threshold (float): The similarity threshold for deduplication. |
| 36 | |
| 37 | Returns: |
| 38 | list: A list of removed duplicate slides. |
| 39 | """ |
| 40 | text_embeddings = get_text_embedding( |
| 41 | [i.to_text() for i in presentation.slides], model, batchsize |
| 42 | ) |
| 43 | pre_embedding = text_embeddings[0] |
| 44 | slide_idx = 1 |
| 45 | duplicates = [] |
| 46 | while slide_idx < len(presentation): |
| 47 | cur_embedding = text_embeddings[slide_idx] |
| 48 | if torch.cosine_similarity(pre_embedding, cur_embedding, -1) > threshold: |
| 49 | duplicates.append(slide_idx - 1) |
| 50 | slide_idx += 1 |
| 51 | pre_embedding = cur_embedding |
| 52 | return [presentation.slides.pop(i) for i in reversed(duplicates)] |
| 53 | |
| 54 | |
| 55 | def get_text_model(device: str = None) -> BGEM3FlagModel: |
no test coverage detected