Function char_based_tokenize

examples/slm_basic_train_ex.cpp:60–69

For simplicity, we assume each line from shakespeare_text is appended, ignoring them.

// For simplicity, we assume each line from shakespeare_text is appended, ignoring them.
std::vector<int> char_based_tokenize(const std::string& text)
{
    std::vector<int> tokens;
    tokens.reserve(text.size());
    for (const int c : text)
    {
        tokens.push_back(std::min(c, MAX_TOKEN_ID));
    }
    return tokens;
}

// Function to shuffle samples and labels in sync
void shuffle_samples_and_labels(std::vector<matrix<int, 0, 1>>& samples, std::vector<unsigned long>& labels) {