Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`.
| 7757 | // Decodes a UTF-8 string which may end in an incomplete sequence. Adds a terminating 0 for use as |
| 7758 | // pointer. If an invalid sequence is encountered, returns `llama_partial_utf8.n_remain == -1`. |
| 7759 | static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8( |
| 7760 | const char * src, |
| 7761 | llama_partial_utf8 partial_start) { |
| 7762 | static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 }; |
| 7763 | const char * pos = src; |
| 7764 | std::vector<uint32_t> code_points; |
| 7765 | uint32_t value = partial_start.value; |
| 7766 | int n_remain = partial_start.n_remain; |
| 7767 | |
| 7768 | // continue previous decode, if applicable |
| 7769 | while (*pos != 0 && n_remain > 0) { |
| 7770 | uint8_t next_byte = static_cast<uint8_t>(*pos); |
| 7771 | if ((next_byte >> 6) != 2) { |
| 7772 | // invalid sequence, abort |
| 7773 | code_points.push_back(0); |
| 7774 | return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, -1 }); |
| 7775 | } |
| 7776 | value = (value << 6) + (next_byte & 0x3F); |
| 7777 | ++pos; |
| 7778 | --n_remain; |
| 7779 | } |
| 7780 | |
| 7781 | if (partial_start.n_remain > 0 && n_remain == 0) { |
| 7782 | code_points.push_back(value); |
| 7783 | } |
| 7784 | |
| 7785 | // decode any subsequent utf-8 sequences, which may end in an incomplete one |
| 7786 | while (*pos != 0) { |
| 7787 | uint8_t first_byte = static_cast<uint8_t>(*pos); |
| 7788 | uint8_t highbits = first_byte >> 4; |
| 7789 | n_remain = lookup[highbits] - 1; |
| 7790 | |
| 7791 | if (n_remain < 0) { |
| 7792 | // invalid sequence, abort |
| 7793 | code_points.clear(); |
| 7794 | code_points.push_back(0); |
| 7795 | return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, n_remain }); |
| 7796 | } |
| 7797 | |
| 7798 | uint8_t mask = (1 << (7 - n_remain)) - 1; |
| 7799 | value = first_byte & mask; |
| 7800 | ++pos; |
| 7801 | while (*pos != 0 && n_remain > 0) { |
| 7802 | value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F); |
| 7803 | ++pos; |
| 7804 | --n_remain; |
| 7805 | } |
| 7806 | if (n_remain == 0) { |
| 7807 | code_points.push_back(value); |
| 7808 | } |
| 7809 | } |
| 7810 | code_points.push_back(0); |
| 7811 | |
| 7812 | return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain }); |
| 7813 | } |
| 7814 | |
| 7815 | // returns true iff pos points to the end of one of the definitions of a rule |
| 7816 | static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { |
no test coverage detected