Function LoadBPEFile

src/main.cpp:89–110

void LoadBPEFile(const std::string& filename, std::vector<VocabItem>& vocab) {
    std::ifstream file(filename);
    std::string line;
    
    while (std::getline(file, line)) {
        if (line.empty()) continue;
        
        std::istringstream iss(line);
        std::string base64_token;
        int rank;
        
        if (iss >> base64_token >> rank) {
            // Decode base64 to bytes
            std::vector<unsigned char> token_bytes = base64_decode(base64_token);
            
            VocabItem item;
            item.rank = rank;
            item.token_bytes = token_bytes;
            vocab.push_back(item);
        }
    }
}

void LoadTokenizer(const std::string& tokenizer_path, const std::string& bpe_path, Llama4Tokenizer& tokenizer) {
    // hard code it for now.