()
| 151 | |
| 152 | |
| 153 | def main(): |
| 154 | if len(sys.argv) < 2: |
| 155 | print(f"Usage: {sys.argv[0]} <tokenizer_dir>", file=sys.stderr) |
| 156 | sys.exit(1) |
| 157 | |
| 158 | tok_dir = sys.argv[1] |
| 159 | vocab_path = os.path.join(tok_dir, "vocab.json") |
| 160 | merges_path = os.path.join(tok_dir, "merges.txt") |
| 161 | |
| 162 | tokenizer = SimpleTokenizer(vocab_path, merges_path) |
| 163 | print(f"Loaded {len(tokenizer.encoder)} vocab entries, " |
| 164 | f"{len(tokenizer.bpe_ranks)} merges") |
| 165 | |
| 166 | print("\n// Expected token IDs (paste into test_tokenizer.cpp if needed):") |
| 167 | for i, text in enumerate(TEST_STRINGS): |
| 168 | tokens = tokenizer.encode(text, context_length=32) |
| 169 | # Print compact |
| 170 | non_zero = [t for t in tokens if t != 0] |
| 171 | pad_count = tokens.count(0) |
| 172 | print(f'// [{i}] "{text}"') |
| 173 | print(f"// → {non_zero} + {pad_count} zeros") |
| 174 | print(f"// full: {tokens}") |
| 175 | print() |
| 176 | |
| 177 | |
| 178 | if __name__ == "__main__": |
no test coverage detected