Normalize a BPE token to the form our runtime tokenizer produces.
(token_str: str)
| 95 | |
| 96 | |
| 97 | def clean_token(token_str: str) -> str: |
| 98 | """Normalize a BPE token to the form our runtime tokenizer produces.""" |
| 99 | s = token_str.strip() |
| 100 | # Strip BPE space markers |
| 101 | s = s.lstrip("\u0120\u2581") |
| 102 | # Strip leading/trailing underscores |
| 103 | s = s.strip("_") |
| 104 | # Lowercase (our runtime tokenizer lowercases) |
| 105 | s = s.lower() |
| 106 | return s |
| 107 | |
| 108 | |
| 109 | # ── Simulated attention ────────────────────────────────────────────── |