Initialize both tokenizers with identical configuration.
(self)
| 186 | return special_tokens |
| 187 | |
| 188 | def setup_tokenizers(self): |
| 189 | """Initialize both tokenizers with identical configuration.""" |
| 190 | print("Setting up tokenizers...") |
| 191 | |
| 192 | # Load configuration based on tokenizer type |
| 193 | if self.tokenizer_type == "llama": |
| 194 | print("Using Llama 4 configuration...") |
| 195 | pattern, vocab, special_tokens = self.load_llama_config() |
| 196 | elif self.tokenizer_type == "mistral": |
| 197 | print("Using Mistral Tekken 7 configuration...") |
| 198 | pattern, vocab, special_tokens = self.load_mistral_config() |
| 199 | else: |
| 200 | raise ValueError(f"Unsupported tokenizer type: {self.tokenizer_type}") |
| 201 | |
| 202 | # Convert TokenDagger format to TikToken format |
| 203 | mergeable_ranks = {} |
| 204 | for item in vocab: |
| 205 | if isinstance(item["token_bytes"], list): |
| 206 | token_bytes = bytes(item["token_bytes"]) |
| 207 | else: |
| 208 | token_bytes = item["token_bytes"] |
| 209 | mergeable_ranks[token_bytes] = item["rank"] |
| 210 | |
| 211 | # Add special tokens to mergeable_ranks |
| 212 | for token_str, rank in special_tokens.items(): |
| 213 | mergeable_ranks[token_str.encode('utf-8')] = rank |
| 214 | |
| 215 | tokenizer_name = f"{self.tokenizer_type}_throughput_test" |
| 216 | |
| 217 | # Initialize TokenDagger using TikToken-compatible API |
| 218 | self.tokendagger_tokenizer = tokendagger.Encoding( |
| 219 | name=tokenizer_name, |
| 220 | pat_str=pattern, |
| 221 | mergeable_ranks=mergeable_ranks, |
| 222 | special_tokens=special_tokens |
| 223 | ) |
| 224 | |
| 225 | # Initialize TikToken with the same configuration |
| 226 | self.tiktoken_tokenizer = tiktoken.Encoding( |
| 227 | name=tokenizer_name, |
| 228 | pat_str=pattern, |
| 229 | mergeable_ranks=mergeable_ranks, |
| 230 | special_tokens=special_tokens |
| 231 | ) |
| 232 | |
| 233 | # Initialize Hugging Face Fast Tokenizer |
| 234 | # Use the appropriate HF model based on tokenizer type |
| 235 | if self.tokenizer_type == "llama": |
| 236 | model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct" |
| 237 | elif self.tokenizer_type == "mistral": |
| 238 | model_name = "mistralai/Ministral-8B-Instruct-2410" |
| 239 | |
| 240 | self.hf_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True) |
| 241 | print(f"✓ HF Fast Tokenizer initialized ({model_name})") |
| 242 | |
| 243 | print(f"✓ TokenDagger tokenizer initialized ({self.tokenizer_type})") |
| 244 | print(f"✓ TikToken tokenizer initialized ({self.tokenizer_type})") |
| 245 |
no test coverage detected