Function read_enwiki

examples/slm_advanced_train_ex.cpp:515–529

Function to read the "enwiki" file (entire or portion)

// Function to read the "enwiki" file (entire or portion)
std::string read_enwiki(const std::string& filepath, size_t max_bytes = 0) {
    std::ifstream file(filepath, std::ios::binary);
    if (!file) {
        throw std::runtime_error("Cannot open enwiki file: " + filepath);
    }
    size_t file_size = get_file_size(filepath);

    // If max_bytes is specified and valid, limit the reading
    size_t bytes_to_read = (max_bytes > 0 && max_bytes < file_size) ? max_bytes : file_size;

    std::string content(bytes_to_read, ' ');
    file.read(&content[0], bytes_to_read);

    return content;
}

// Function to verify byte-for-byte matching with detailed error reporting
bool verify_match(const std::string& original, const std::string& generated) {