Class MistralTokenizer

src/mistral_main.cpp:58–74

}

struct MistralTokenizer {
    TokenizerConfig config;
    std::vector<VocabItem> vocab;
    std::unique_ptr<tiktoken::CoreBPE> bpe;

    std::vector<int> encode(const std::string& prompt) const {
        std::vector<int> tokens = {MistralSpecialTokens::BOS, MistralSpecialTokens::BEGIN_INST};
        if (bpe) {
            auto result = bpe->encode_ordinary(prompt);
            for (int token : result) {
                tokens.push_back(token + config.default_num_special_tokens);
            }
        }
        tokens.push_back(MistralSpecialTokens::END_INST);
        return tokens;
    }
};

NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TokenizerConfig, pattern, num_vocab_tokens, default_vocab_size, default_num_special_tokens, version);
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(VocabItem, rank, token_bytes, token_str);