| 3405 | } |
| 3406 | |
| 3407 | llama_token llama_vocab::byte_to_token(uint8_t ch) const { |
| 3408 | GGML_ASSERT(get_type() != LLAMA_VOCAB_TYPE_NONE); |
| 3409 | static const char * hex = "0123456789ABCDEF"; |
| 3410 | switch (get_type()) { |
| 3411 | case LLAMA_VOCAB_TYPE_SPM: |
| 3412 | case LLAMA_VOCAB_TYPE_UGM: { |
| 3413 | const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 }; |
| 3414 | auto token = pimpl->token_to_id.find(buf); |
| 3415 | if (token != pimpl->token_to_id.end()) { |
| 3416 | return (*token).second; |
| 3417 | } |
| 3418 | // Try to fall back to just the byte as a string |
| 3419 | const char buf2[2] = { (char)ch, 0 }; |
| 3420 | return pimpl->token_to_id.at(buf2); |
| 3421 | } |
| 3422 | case LLAMA_VOCAB_TYPE_WPM: |
| 3423 | case LLAMA_VOCAB_TYPE_BPE: { |
| 3424 | return pimpl->token_to_id.at(unicode_byte_to_utf8(ch)); |
| 3425 | } |
| 3426 | case LLAMA_VOCAB_TYPE_PLAMO2: { |
| 3427 | // PLaMo-2 uses byte tokens in format <0xXX> |
| 3428 | char hex_str[8]; |
| 3429 | snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch); |
| 3430 | return pimpl->token_to_id.at(hex_str); |
| 3431 | } |
| 3432 | default: |
| 3433 | GGML_ABORT("fatal error"); |
| 3434 | } |
| 3435 | } |
| 3436 | |
| 3437 | llama_token llama_vocab::text_to_token(const std::string & text) const { |
| 3438 | GGML_ASSERT(pimpl->type != LLAMA_VOCAB_TYPE_NONE); |