Function LoadTokenizer

src/mistral_main.cpp:99–146

void LoadTokenizer(const std::string& filename, MistralTokenizer& tokenizer) {
    std::ifstream file(filename);
    nlohmann::json json_data;
    file >> json_data;
    
    // Parse config
    tokenizer.config = json_data["config"].get<TokenizerConfig>();
    
    // Parse vocab with custom handling for optional token_str
    tokenizer.vocab.clear();

    int vocab_size = tokenizer.config.default_vocab_size - tokenizer.config.default_num_special_tokens;

    for (size_t i = 0; i < vocab_size; i++) {
        const auto& vocab_item = json_data["vocab"][i];
        VocabItem item;
        item.rank = vocab_item["rank"].get<int>();
        
        std::string token_bytes_str = vocab_item["token_bytes"].get<std::string>();
        
        // Decode base64 string to binary data
        std::string decoded_bytes;
        try {
            decoded_bytes = base64_decode(token_bytes_str);
        } catch (const std::exception& e) {
            fprintf(stderr, "ERROR: Failed to decode base64 token_bytes: %s\n", e.what());
            continue; // Skip this vocab item
        }
        
        item.token_bytes = std::vector<unsigned char>(decoded_bytes.begin(), decoded_bytes.end());
        
        // Handle optional token_str field
        if (vocab_item.contains("token_str") && !vocab_item["token_str"].is_null()) {
            item.token_str = vocab_item["token_str"].get<std::string>();
        } else {
            item.token_str = ""; // Default to empty string if not present or null
        }
        
        tokenizer.vocab.push_back(item);
    }
    
    // Create the BPE tokenizer and initialize it
    tokenizer.bpe = std::make_unique<tiktoken::CoreBPE>(tokenizer.vocab);
    if (!tokenizer.bpe->init_regex(tokenizer.config.pattern)) {
        fprintf(stderr, "ERROR: Failed to initialize regex for tokenizer\n");
        tokenizer.bpe.reset(); // Clear the pointer
    }
}


void Tokenize(const MistralTokenizer& tokenizer, const std::string& prompt) {