Method encode

src/main.cpp:50–63

    std::unique_ptr<tiktoken::CoreBPE> bpe;

    std::vector<int> encode(const std::string& prompt) const {
        // std::vector<int> tokens = {MistralSpecialTokens::BOS, MistralSpecialTokens::BEGIN_INST};
        std::vector<int> tokens = {Llama4SpecialTokens::BOS.rank};
        tokens.reserve(prompt.size()); // some compression occurs, so shouldn't be greater than the prompt size.
        if (bpe) {
            // auto result = bpe->encode_ordinary(prompt);
            auto result = bpe->encode(prompt, {Llama4SpecialTokens::BOS.content, Llama4SpecialTokens::EOS.content, Llama4SpecialTokens::FULL_EOS.content});
            for (auto token : result.first) {
                tokens.push_back(token);
            }
        }
        // tokens.push_back(MistralSpecialTokens::END_INST);
        return tokens;
    }
};

NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(SpecialToken, content);