(token_ids: list[int])
| 49 | return torch.tensor(ids, dtype=torch.long) |
| 50 | |
| 51 | def int_to_text(token_ids: list[int]) -> str: |
| 52 | # Collapse repeats & remove blank (0) |
| 53 | pieces, prev = [], None |
| 54 | for t in token_ids: |
| 55 | if t == BLANK: |
| 56 | prev = None |
| 57 | continue |
| 58 | if t == prev: # ← add this |
| 59 | continue |
| 60 | raw = t - 1 |
| 61 | pieces.append(raw) |
| 62 | prev = t |
| 63 | return sp.decode(pieces) |
| 64 | |
| 65 | |
| 66 | # ----------------------------------------------------------------------------- |