Multithreaded throughput benchmark suite.
| 62 | |
| 63 | |
| 64 | class ThroughputBenchmark: |
| 65 | """Multithreaded throughput benchmark suite.""" |
| 66 | |
| 67 | def __init__(self, |
| 68 | tokenizer_type: str = "llama", |
| 69 | thread_counts: List[int] = [1, 2, 4, 8, 16, 32], |
| 70 | text_size_mb: int = 1024, # 1GB by default |
| 71 | iterations_per_thread: int = 10): |
| 72 | self.src_dir = Path(__file__).parent.parent / "src" |
| 73 | self.test_dir = Path(__file__).parent |
| 74 | self.tokenizer_type = tokenizer_type.lower() |
| 75 | self.thread_counts = thread_counts |
| 76 | self.text_size_mb = text_size_mb |
| 77 | self.iterations_per_thread = iterations_per_thread |
| 78 | self.results: List[ThroughputResult] = [] |
| 79 | |
| 80 | # Validate tokenizer type |
| 81 | if self.tokenizer_type not in ["llama", "mistral"]: |
| 82 | raise ValueError(f"Invalid tokenizer type: {tokenizer_type}. Must be 'llama' or 'mistral'") |
| 83 | |
| 84 | print(f"Throughput benchmark configuration:") |
| 85 | print(f" Tokenizer: {self.tokenizer_type}") |
| 86 | print(f" Thread counts: {self.thread_counts}") |
| 87 | print(f" Text size: {self.text_size_mb} MB") |
| 88 | print(f" Iterations per thread: {self.iterations_per_thread}") |
| 89 | |
| 90 | def load_llama_config(self) -> Tuple[str, List[Dict[str, Any]], Dict[str, int]]: |
| 91 | """ |
| 92 | Load Llama 4 configuration from the codebase. |
| 93 | https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct |
| 94 | """ |
| 95 | # Hard-coded pattern from main.cpp |
| 96 | pattern = r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+" |
| 97 | |
| 98 | # Load vocabulary from tokenizer.model |
| 99 | vocab = self.load_bpe_vocab() |
| 100 | |
| 101 | # Load special tokens from tokenizer_config.json |
| 102 | special_tokens = self.load_special_tokens() |
| 103 | |
| 104 | return pattern, vocab, special_tokens |
| 105 | |
| 106 | def load_mistral_config(self) -> Tuple[str, List[Dict[str, Any]], Dict[str, int]]: |
| 107 | """ |
| 108 | Load Mistral's Tekken 7 configuration from the codebase. |
| 109 | https://huggingface.co/mistralai/Ministral-8B-Instruct-2410/tree/main |
| 110 | """ |
| 111 | config_file = self.test_dir / "configs" / "mistral3.2" / "tekken.json" |
| 112 | if not config_file.exists(): |
| 113 | raise FileNotFoundError(f"Tekken config file not found: {config_file}") |
| 114 | |
| 115 | with open(config_file, 'r', encoding='utf-8') as f: |
| 116 | config = json.load(f) |
| 117 | |
| 118 | # Extract pattern from config |
| 119 | pattern = config["config"]["pattern"] |
| 120 | |
| 121 | # Extract vocabulary |