The words of a message are the pieces between whitespace and ASCII punctuation.
| 202 | |
| 203 | // The words of a message are the pieces between whitespace and ASCII punctuation. |
| 204 | static std::vector<string> tokenize(const string& message) |
| 205 | { |
| 206 | return split(dlib::tolower(message), " \t\n\r\v\f!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"); |
| 207 | } |
| 208 | |
| 209 | static std::pair<sample_type, sample_type> count_ngrams(const std::vector<string>& tokens) |
| 210 | { |