High-level interface for the tokendagger tokenizer. This class provides a tiktoken-compatible API around the tokendagger.CoreBPE class.
| 26 | |
| 27 | |
| 28 | class Tokenizer: |
| 29 | """High-level interface for the tokendagger tokenizer. |
| 30 | |
| 31 | This class provides a tiktoken-compatible API around the tokendagger.CoreBPE class. |
| 32 | """ |
| 33 | |
| 34 | def __init__( |
| 35 | self, |
| 36 | name: str, |
| 37 | *, |
| 38 | pattern: str | None = None, |
| 39 | pat_str: str | None = None, # for tiktoken compat. |
| 40 | vocab: list[dict] | None = None, |
| 41 | mergeable_ranks: dict[bytes, int] | None = None, # for tiktoken compat. |
| 42 | special_tokens: dict[str, int] | None = None, |
| 43 | vocab_file: str | Path | None = None, |
| 44 | special_tokens_file: str | Path | None = None, |
| 45 | ): |
| 46 | """Initialize the tokenizer. |
| 47 | |
| 48 | Args: |
| 49 | name: Name of the tokenizer |
| 50 | pattern: Regex pattern for text splitting |
| 51 | pat_str: Regex pattern for text splitting (for tiktoken compat) |
| 52 | vocab: List of vocabulary items as dicts with 'rank', 'token_bytes', 'token_string' |
| 53 | mergeable_ranks: Dict mapping bytes to their ranks (for tiktoken compat) |
| 54 | special_tokens: Dict mapping special token strings to their IDs |
| 55 | vocab_file: Path to vocabulary file (JSON format) |
| 56 | special_tokens_file: Path to special tokens file (JSON format) |
| 57 | """ |
| 58 | self.name = name |
| 59 | # TikToken compatibility |
| 60 | if pat_str is not None: |
| 61 | pattern = pat_str |
| 62 | if mergeable_ranks is not None: |
| 63 | vocab = mergeable_ranks |
| 64 | self.pattern = pattern |
| 65 | |
| 66 | # Convert TikToken format to TokenDagger format if needed |
| 67 | if isinstance(mergeable_ranks, dict): |
| 68 | # TikToken format: {bytes: rank} -> TokenDagger format |
| 69 | vocab_list = [] |
| 70 | for token_bytes, rank in mergeable_ranks.items(): |
| 71 | vocab_list.append({ |
| 72 | "rank": rank, |
| 73 | "token_bytes": list(token_bytes), |
| 74 | "token_string": "" |
| 75 | }) |
| 76 | vocab = vocab_list |
| 77 | |
| 78 | # Load vocabulary |
| 79 | if vocab_file: |
| 80 | vocab = self._load_vocab_file(vocab_file) |
| 81 | elif vocab is None: |
| 82 | raise ValueError("Either 'vocab', 'mergeable_ranks', or 'vocab_file' must be provided") |
| 83 | |
| 84 | # Load special tokens |
| 85 | if special_tokens_file: |
no outgoing calls
no test coverage detected