| 24 | #pragma GCC diagnostic pop |
| 25 | |
| 26 | struct TokenizerConfig { |
| 27 | std::string pattern; |
| 28 | int num_vocab_tokens; |
| 29 | int default_vocab_size; |
| 30 | int default_num_special_tokens; |
| 31 | std::string version; |
| 32 | }; |
| 33 | |
| 34 | // Special token IDs as constants |
| 35 | namespace MistralSpecialTokens { |
nothing calls this directly
no outgoing calls
no test coverage detected