| 159 | } |
| 160 | |
| 161 | static std::pair<uint32_t, const char *> parse_char(const char * src) { |
| 162 | if (*src == '\\') { |
| 163 | switch (src[1]) { |
| 164 | case 'x': return parse_hex(src + 2, 2); |
| 165 | case 'u': return parse_hex(src + 2, 4); |
| 166 | case 'U': return parse_hex(src + 2, 8); |
| 167 | case 't': return std::make_pair('\t', src + 2); |
| 168 | case 'r': return std::make_pair('\r', src + 2); |
| 169 | case 'n': return std::make_pair('\n', src + 2); |
| 170 | case '\\': |
| 171 | case '"': |
| 172 | case '[': |
| 173 | case ']': |
| 174 | return std::make_pair(src[1], src + 2); |
| 175 | default: |
| 176 | throw std::runtime_error(std::string("unknown escape at ") + src); |
| 177 | } |
| 178 | } else if (*src) { |
| 179 | return decode_utf8(src); |
| 180 | } |
| 181 | throw std::runtime_error("unexpected end of input"); |
| 182 | } |
| 183 | |
| 184 | static std::pair<uint32_t, const char *> parse_token(const llama_vocab * vocab, const char * src) { |
| 185 | const char * pos = src; |
no test coverage detected