Function pair_key

src/framework/tokenizers/llama_bpe.cpp:78–83

}

std::string pair_key(const std::string & left, const std::string & right) {
    std::string key = left;
    key.push_back('\0');
    key += right;
    return key;
}

uint32_t token_attr_from_json(const engine::io::json::Value & token_config) {
    // HF added tokens must be matched as whole tokens during encode even when