Adds ngram to the names of its dimension, if tfidf kept its bucket. The bucket is the one hash_word_ngrams() and hash_char_ngrams() count it in, and since vocabulary() lists the kept buckets in increasing order, its position there is the dimension, counted from offset.
| 172 | // vocabulary() lists the kept buckets in increasing order, its position there is the |
| 173 | // dimension, counted from offset. |
| 174 | void add_name( |
| 175 | const tfidf_transformer<sample_type>& tfidf, |
| 176 | const unsigned long offset, |
| 177 | const string& ngram |
| 178 | ) |
| 179 | { |
| 180 | const unsigned long b = murmur_hash3(ngram.data(), ngram.size(), 0) % num_buckets; |
| 181 | const auto& vocabulary = tfidf.vocabulary(); |
| 182 | const auto it = std::lower_bound(vocabulary.begin(), vocabulary.end(), b); |
| 183 | if (it != vocabulary.end() && *it == b) |
| 184 | names[offset + (it - vocabulary.begin())].insert(ngram); |
| 185 | } |
| 186 | |
| 187 | static constexpr unsigned long num_buckets = 1 << 20; |
| 188 | static constexpr unsigned long min_word_n = 1, max_word_n = 2; |