Class CodePerformanceBenchmark

tests/code_performance_benchmark.py:64–590

Performance benchmark specifically for code tokenization.

class CodePerformanceBenchmark:
    """Performance benchmark specifically for code tokenization."""
    
    def __init__(self, warmup_runs: int = 3, benchmark_runs: int = 25, tokenizer_type: str = "llama"):
        self.repo_root = Path(__file__).parent.parent
        self.warmup_runs = warmup_runs
        self.benchmark_runs = benchmark_runs
        self.tokenizer_type = tokenizer_type.lower()
        self.results: List[CodeBenchmarkResult] = []
        
        # Validate tokenizer type
        if self.tokenizer_type not in ["llama", "mistral"]:
            raise ValueError(f"Invalid tokenizer type: {tokenizer_type}. Must be 'llama' or 'mistral'")
        
        # Code file extensions to test
        self.code_extensions = {
            '.py': 'Python',
            '.cpp': 'C++',
            '.c': 'C',
            '.h': 'C Header',
            '.hpp': 'C++ Header',
            '.js': 'JavaScript',
            '.ts': 'TypeScript',
            '.java': 'Java',
            '.rs': 'Rust',
            '.go': 'Go',
            '.rb': 'Ruby',
            '.php': 'PHP',
            '.cs': 'C#',
            '.swift': 'Swift',
            '.kt': 'Kotlin',
            '.scala': 'Scala',
            '.sh': 'Shell',
            '.bat': 'Batch',
            '.ps1': 'PowerShell',
            '.sql': 'SQL',
            '.json': 'JSON',
            '.xml': 'XML',
            '.yaml': 'YAML',
            '.yml': 'YAML',
            '.md': 'Markdown',
            '.txt': 'Text',
            '.makefile': 'Makefile',
            '.cmake': 'CMake'
        }
        
    def setup_tokenizers(self):
        """Initialize both tokenizers."""
        print(f"Setting up tokenizers with {self.tokenizer_type.title()} configuration...")
        
        # Load configuration based on tokenizer type
        if self.tokenizer_type == "llama":
            pattern, vocab, special_tokens = self.load_llama_config()
        elif self.tokenizer_type == "mistral":
            pattern, vocab, special_tokens = self.load_mistral_config()
        else:
            raise ValueError(f"Unsupported tokenizer type: {self.tokenizer_type}")