| 65 | // ---------------------------------------------------------------------------------------- |
| 66 | |
| 67 | void test_term_selection() |
| 68 | { |
| 69 | tfidf_transformer<sparse_type> tfidf; |
| 70 | tfidf.set_min_document_frequency(2); |
| 71 | tfidf.train(corpus()); |
| 72 | DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({1})); |
| 73 | const sparse_type r = tfidf(corpus()[0]); |
| 74 | DLIB_TEST(r.size() == 1 && r[0].first == 0 && near(r[0].second, 1)); |
| 75 | |
| 76 | // The most frequent terms by total count: term 1 (4), then term 2 (3). |
| 77 | tfidf.set_min_document_frequency(1); |
| 78 | tfidf.set_max_vocabulary_size(2); |
| 79 | tfidf.train(corpus()); |
| 80 | DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({1,2})); |
| 81 | |
| 82 | // Terms 0 and 3 tie with a total count of 1; the smaller term wins. |
| 83 | tfidf.set_max_vocabulary_size(3); |
| 84 | tfidf.train(corpus()); |
| 85 | DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({0,1,2})); |
| 86 | |
| 87 | tfidf.set_max_vocabulary_size(0); |
| 88 | tfidf.train(corpus()); |
| 89 | DLIB_TEST(tfidf.vocabulary_size() == 0); |
| 90 | DLIB_TEST(tfidf(corpus()[0]).empty()); |
| 91 | } |
| 92 | |
| 93 | // ---------------------------------------------------------------------------------------- |
| 94 |
no test coverage detected