| 89 | { |
| 90 | public: |
| 91 | void train(const std::vector<string>& messages) |
| 92 | { |
| 93 | std::vector<std::vector<string>> tokens; |
| 94 | std::vector<sample_type> word_counts, char_counts; |
| 95 | for (const auto& message : messages) |
| 96 | { |
| 97 | tokens.push_back(tokenize(message)); |
| 98 | auto [w, c] = count_ngrams(tokens.back()); |
| 99 | word_counts.push_back(std::move(w)); |
| 100 | char_counts.push_back(std::move(c)); |
| 101 | } |
| 102 | words.train(word_counts); |
| 103 | chars.train(char_counts); |
| 104 | |
| 105 | // The hashes can't be undone, so to tell what a dimension stands for, we record |
| 106 | // which n-grams of the training messages ended up in it. |
| 107 | names.assign(dimensions(), {}); |
| 108 | for (const auto& message_tokens : tokens) |
| 109 | { |
| 110 | for (const auto& ngram : word_ngrams(message_tokens, min_word_n, max_word_n)) |
| 111 | add_name(words, 0, ngram); |
| 112 | for (const auto& ngram : char_ngrams(message_tokens, min_char_n, max_char_n)) |
| 113 | add_name(chars, words.vocabulary_size(), ngram); |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Returns the n-grams of the training messages behind dimension j of operator()'s |
| 118 | // output. There is more than one when their hashes went to the same bucket. |
no test coverage detected