Function byte_pair_encode

src/tiktoken/tiktoken.cpp:371–378

Byte pair encoding

    // Byte pair encoding
    void byte_pair_encode(const std::vector<unsigned char>& piece, const emhash8::HashMap<std::vector<unsigned char>, int, VectorHashEmhash>& encoder, std::vector<int>& result) {
        if (piece.size() == 1) {
            result.push_back(encoder.at(piece));
            return;
        }
        // TODO: stuff
        bpe_merge(piece, encoder, result);
    }

} // namespace tiktoken