Function LoadTokenizer

src/main.cpp:112–137

}

void LoadTokenizer(const std::string& tokenizer_path, const std::string& bpe_path, Llama4Tokenizer& tokenizer) {
    // hard code it for now.
    std::string pattern_str = "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";

    // load base vocab.
    std::vector<VocabItem> vocab;
    LoadBPEFile(bpe_path, vocab);

    // load special tokens.
    std::ifstream file(tokenizer_path);
    nlohmann::json json_data;
    file >> json_data;
    // Parse config
    tokenizer.config = json_data.get<TokenizerConfig>();
    std::vector<VocabItem> special_vocab;
    for (const auto& [token_str, special_token] : tokenizer.config.added_tokens_decoder) {
        VocabItem item;
        item.rank = std::stoi(token_str);
        item.token_string = special_token.content;
        item.token_bytes = std::vector<unsigned char>(special_token.content.begin(), special_token.content.end());
        special_vocab.push_back(item);
    }

    // Create the BPE tokenizer and initialize it
    tokenizer.bpe = std::make_unique<tiktoken::CoreBPE>(pattern_str, vocab, special_vocab);
}


void Tokenize(const Llama4Tokenizer& tokenizer, const std::string& prompt) {