This object turns a message into the sparse vector we give the SVM. It uses two families of features: - Word unigrams and bigrams, e.g. "free" and "call now". - Character n-grams of 3 to 5 bytes within each word, e.g. " fre", "free", "ree ". They still match misspelled or abbreviated words: "free", "freee" and "freebies" share some of them. hash_word_ngrams() and hash_char_ngrams() hash each n-g
| 86 | // Each family gets its own tfidf_transformer, since their document frequencies are not |
| 87 | // comparable, and the char dimensions go after the word dimensions. |
| 88 | class text_features |
| 89 | { |
| 90 | public: |
| 91 | void train(const std::vector<string>& messages) |
| 92 | { |
| 93 | std::vector<std::vector<string>> tokens; |
| 94 | std::vector<sample_type> word_counts, char_counts; |
| 95 | for (const auto& message : messages) |
| 96 | { |
| 97 | tokens.push_back(tokenize(message)); |
| 98 | auto [w, c] = count_ngrams(tokens.back()); |
| 99 | word_counts.push_back(std::move(w)); |
| 100 | char_counts.push_back(std::move(c)); |
| 101 | } |
| 102 | words.train(word_counts); |
| 103 | chars.train(char_counts); |
| 104 | |
| 105 | // The hashes can't be undone, so to tell what a dimension stands for, we record |
| 106 | // which n-grams of the training messages ended up in it. |
| 107 | names.assign(dimensions(), {}); |
| 108 | for (const auto& message_tokens : tokens) |
| 109 | { |
| 110 | for (const auto& ngram : word_ngrams(message_tokens, min_word_n, max_word_n)) |
| 111 | add_name(words, 0, ngram); |
| 112 | for (const auto& ngram : char_ngrams(message_tokens, min_char_n, max_char_n)) |
| 113 | add_name(chars, words.vocabulary_size(), ngram); |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Returns the n-grams of the training messages behind dimension j of operator()'s |
| 118 | // output. There is more than one when their hashes went to the same bucket. |
| 119 | const std::set<string>& describe(const unsigned long j) const |
| 120 | { |
| 121 | return names.at(j); |
| 122 | } |
| 123 | |
| 124 | sample_type operator()(const string& message) const |
| 125 | { |
| 126 | const auto [w, c] = count_ngrams(tokenize(message)); |
| 127 | sample_type x = words(w); |
| 128 | for (const auto& [dimension, value] : chars(c)) |
| 129 | x.emplace_back(words.vocabulary_size() + dimension, value); |
| 130 | return x; |
| 131 | } |
| 132 | |
| 133 | unsigned long dimensions() const |
| 134 | { |
| 135 | return words.vocabulary_size() + chars.vocabulary_size(); |
| 136 | } |
| 137 | |
| 138 | // This saves the learned state, i.e. the vocabularies and idf weights. The tokenizer, |
| 139 | // the n-gram ranges and num_buckets are part of the code, so a saved object only makes |
| 140 | // sense to a program built with the same settings. |
| 141 | friend void serialize(const text_features& item, std::ostream& out) |
| 142 | { |
| 143 | serialize(item.words, out); |
| 144 | serialize(item.chars, out); |
| 145 | serialize(item.names, out); |
nothing calls this directly
no test coverage detected