Method cpt_to_utf8

src/common/tokenizer/tokenizer.cpp:84–151

\brief Convert the codepoint to utf8 \param input the input \return the utf8 string

/// \param input the input
/// \return the utf8 string
std::string Tokenizer::cpt_to_utf8(const std::string& input) {
    static const std::string pattern = "▁";
    static const size_t pattern_size = pattern.size();
    static auto inv_map = this->make_inverse_byte_map();
    std::string output = "";
    output.reserve(input.size());
    size_t i = 0;
    if (!this->is_doubled_encoded) { // simply do pattern substitution of "▁" to " ", temporary solution
        std::string output = "";
        for (size_t i = 0; i < input.size(); i++) {
            if (i <= input.size() - pattern_size && input.substr(i, pattern_size) == pattern) {
                output += " ";
                i += pattern_size - 1; // -1 because the loop will increment i by 1
            }
            else {
                output += input[i];
            }
        }
        return output;
    }
    while (i < input.size()) {
        unsigned char c = input[i];
        uint32_t cp;
        size_t width;

        // --- Decode a UTF-8 codepoint ---
        if (c < 0x80) {
            cp = c;
            width = 1;
        }
        else if ((c & 0xE0) == 0xC0) {
            if (i + 1 >= input.size()) throw std::runtime_error("Truncated UTF-8, ll = 2");
            cp  = (uint32_t)(c     & 0x1F) << 6;
            cp |= (uint32_t)(input[i+1] & 0x3F);
            width = 2;
        }
        else if ((c & 0xF0) == 0xE0) {
            if (i + 2 >= input.size()) throw std::runtime_error("Truncated UTF-8, ll = 3");
            cp  = (uint32_t)(c     & 0x0F) << 12;
            cp |= (uint32_t)(input[i+1] & 0x3F) << 6;
            cp |= (uint32_t)(input[i+2] & 0x3F);
            width = 3;
        }
        else if ((c & 0xF8) == 0xF0) {
            if (i + 3 >= input.size()) throw std::runtime_error("Truncated UTF-8, ll = 4");
            cp  = (uint32_t)(c     & 0x07) << 18;
            cp |= (uint32_t)(input[i+1] & 0x3F) << 12;
            cp |= (uint32_t)(input[i+2] & 0x3F) << 6;
            cp |= (uint32_t)(input[i+3] & 0x3F);
            width = 4;
        }
        else {
            std::cout << std::hex <<"cp: " << cp << std::endl;
            throw std::runtime_error("Invalid UTF-8 byte");
        }

        // --- Look up the original byte ---
        auto it = inv_map.find(cp);