Filter vocabulary to code-relevant tokens. Goal: keep tokens that our runtime camelCase/snake_case splitter would produce from identifiers. Reject BPE noise, punctuation combos, and non-Latin scripts.
(token_str: str)
| 56 | # ── Token filtering ─────────────────────────────────────────────────── |
| 57 | |
| 58 | def is_code_relevant(token_str: str) -> bool: |
| 59 | """Filter vocabulary to code-relevant tokens. |
| 60 | |
| 61 | Goal: keep tokens that our runtime camelCase/snake_case splitter would |
| 62 | produce from identifiers. Reject BPE noise, punctuation combos, and |
| 63 | non-Latin scripts. |
| 64 | """ |
| 65 | s = token_str.strip() |
| 66 | if not s: |
| 67 | return False |
| 68 | |
| 69 | # Remove BPE markers (Ġ = space prefix, ▁ = sentencepiece, Ċ/ċ = newline in Qwen) |
| 70 | clean = s.lstrip("\u0120\u2581") # Ġ, ▁ |
| 71 | if not clean: |
| 72 | return False |
| 73 | |
| 74 | # Skip special tokens |
| 75 | if clean.startswith("<") and clean.endswith(">"): |
| 76 | return False |
| 77 | if clean.startswith("[") and clean.endswith("]"): |
| 78 | return False |
| 79 | |
| 80 | # Strip leading/trailing underscores (common in BPE) but keep content |
| 81 | inner = clean.strip("_") |
| 82 | if not inner: |
| 83 | return False |
| 84 | |
| 85 | # STRICT: must be purely alphanumeric + underscores (identifier-shaped) |
| 86 | # This rejects BPE noise like "!");ċ", "!!!!ċċ", etc. |
| 87 | if not re.match(r'^[a-zA-Z][a-zA-Z0-9_]*$', inner): |
| 88 | return False |
| 89 | |
| 90 | # Must be at least 2 chars of actual content |
| 91 | if len(inner) < 2: |
| 92 | return False |
| 93 | |
| 94 | return True |
| 95 | |
| 96 | |
| 97 | def clean_token(token_str: str) -> str: |