Method encode

src/common/tokenizer/tokenizer.cpp:156–158

\brief Encode the text \param text the text \return the encoded tokens

/// \param text the text
/// \return the encoded tokens
std::vector<int> Tokenizer::encode(const std::string& text) {
    return this->tokenizer->Encode(text);
}

/// \brief Decode the tokens
/// \param tokens the tokens