Method setup_tokenizers

tests/performance_benchmark.py:191–237

Initialize both tokenizers with identical configuration.

(self)

        return special_tokens
    
    def setup_tokenizers(self):
        """Initialize both tokenizers with identical configuration."""
        print("Setting up tokenizers...")
        
        # Load configuration based on tokenizer type
        if self.tokenizer_type == "llama":
            print("Using Llama 4 configuration...")
            pattern, vocab, special_tokens = self.load_llama_config()
        elif self.tokenizer_type == "mistral":
            print("Using Mistral Tekken 7 configuration...")
            pattern, vocab, special_tokens = self.load_mistral_config()
        else:
            raise ValueError(f"Unsupported tokenizer type: {self.tokenizer_type}")
        
        # Convert TokenDagger format to TikToken format
        mergeable_ranks = {}
        for item in vocab:
            if isinstance(item["token_bytes"], list):
                token_bytes = bytes(item["token_bytes"])
            else:
                token_bytes = item["token_bytes"]
            mergeable_ranks[token_bytes] = item["rank"]
        
        # Add special tokens to mergeable_ranks
        for token_str, rank in special_tokens.items():
            mergeable_ranks[token_str.encode('utf-8')] = rank
        
        tokenizer_name = f"{self.tokenizer_type}_perf_test"
        
        # Initialize TokenDagger using TikToken-compatible API
        self.tokendagger_tokenizer = tokendagger.Encoding(
            name=tokenizer_name,
            pat_str=pattern,
            mergeable_ranks=mergeable_ranks,
            special_tokens=special_tokens
        )
        
        # Initialize TikToken with the same configuration
        self.tiktoken_tokenizer = tiktoken.Encoding(
            name=tokenizer_name,
            pat_str=pattern,
            mergeable_ranks=mergeable_ranks,
            special_tokens=special_tokens
        )
        
        print(f"✓ TokenDagger tokenizer initialized ({self.tokenizer_type})")
        print(f"✓ TikToken tokenizer initialized ({self.tokenizer_type})")
    
    def generate_test_texts(self) -> Dict[str, List[str]]:
        """Generate comprehensive test corpus with various edge cases."""