Method encode_ordinary

src/tiktoken/tiktoken.cpp:156–167

    }

    std::vector<int> CoreBPE::encode_ordinary(const std::string& text) const {
        std::vector<int> result;
        // Split text using regex
        auto pieces = split_text(text, 0, text.length());
        for (const auto& piece : pieces) {
            std::vector<unsigned char> bytes(piece.begin(), piece.end());
            // TODO: check if the bytes exist in the encoder, direct lookup before falling back to BPE.
            byte_pair_encode(bytes, encoder, result);
        }
        
        return result;
    }

    std::pair<std::vector<int>, int> CoreBPE::encode(const std::string& text, const emhash8::HashSet<std::string>& allowed_special) {
        std::vector<int> result;