| 259 | } |
| 260 | |
| 261 | std::string Tokenizer::get_default_stop_sequence() const { |
| 262 | switch (model_type_) { |
| 263 | case ModelType::GEMMA: |
| 264 | return "<end_of_turn>"; |
| 265 | case ModelType::GEMMA4: |
| 266 | return "<turn|>"; |
| 267 | case ModelType::QWEN: |
| 268 | case ModelType::QWEN3P5: |
| 269 | case ModelType::LFM2: |
| 270 | return "<|im_end|>"; |
| 271 | case ModelType::NEEDLE: |
| 272 | return ""; |
| 273 | default: |
| 274 | return "<|im_end|>"; |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | std::vector<uint32_t> Tokenizer::apply_chat_template(const std::vector<ChatMessage>& messages, bool add_generation_prompt) const { |
| 279 | std::string formatted_prompt = format_chat_prompt(messages, add_generation_prompt); |
no outgoing calls
no test coverage detected