| 189 | static constexpr unsigned long min_char_n = 3, max_char_n = 5; |
| 190 | |
| 191 | static tfidf_transformer<sample_type> make_tfidf() |
| 192 | { |
| 193 | tfidf_transformer<sample_type> tfidf; |
| 194 | // Ignore n-grams that appear in a single training message: they can't help with |
| 195 | // messages the model hasn't seen, and they make the vectors longer. |
| 196 | tfidf.set_min_document_frequency(2); |
| 197 | // A word repeated 10 times in a message is not 10 times more telling than a word |
| 198 | // that appears once, so we weigh a count c as 1 + log(c). |
| 199 | tfidf.use_sublinear_tf(true); |
| 200 | return tfidf; |
| 201 | } |
| 202 | |
| 203 | // The words of a message are the pieces between whitespace and ASCII punctuation. |
| 204 | static std::vector<string> tokenize(const string& message) |
nothing calls this directly
no test coverage detected