| 72 | } |
| 73 | |
| 74 | std::string byte_fallback_token(unsigned char byte) { |
| 75 | constexpr char kHex[] = "0123456789ABCDEF"; |
| 76 | std::string out = "<0x"; |
| 77 | out.push_back(kHex[(byte >> 4U) & 0x0FU]); |
| 78 | out.push_back(kHex[byte & 0x0FU]); |
| 79 | out.push_back('>'); |
| 80 | return out; |
| 81 | } |
| 82 | |
| 83 | std::vector<std::string> bpe_initial_pieces( |
| 84 | std::string_view normalized_text, |
no outgoing calls
no test coverage detected