| 93 | // ---------------------------------------------------------------------------------------- |
| 94 | |
| 95 | void test_weighting_options() |
| 96 | { |
| 97 | tfidf_transformer<sparse_type> tfidf; |
| 98 | tfidf.normalize_to_unit_length(false); |
| 99 | tfidf.train(corpus()); |
| 100 | |
| 101 | sparse_type r = tfidf(corpus()[1]); |
| 102 | DLIB_TEST(r.size() == 2); |
| 103 | DLIB_TEST(r[0].first == 1 && near(r[0].second, 1*1.0)); |
| 104 | DLIB_TEST(r[1].first == 2 && near(r[1].second, 3*(std::log(2.0)+1))); |
| 105 | |
| 106 | // Weighting options apply to operator() right away. |
| 107 | tfidf.use_sublinear_tf(true); |
| 108 | r = tfidf(corpus()[1]); |
| 109 | DLIB_TEST(near(r[0].second, (1 + std::log(1.0))*1)); |
| 110 | DLIB_TEST(near(r[1].second, (1 + std::log(3.0))*(std::log(2.0)+1))); |
| 111 | |
| 112 | // idf smoothing applies on the next train(). |
| 113 | tfidf.use_smooth_idf(false); |
| 114 | DLIB_TEST(near(tfidf.idf()[0], std::log(2.0) + 1)); |
| 115 | tfidf.train(corpus()); |
| 116 | DLIB_TEST(near(tfidf.idf()[0], std::log(3.0) + 1)); |
| 117 | DLIB_TEST(near(tfidf.idf()[1], 1)); |
| 118 | |
| 119 | // So does term selection. |
| 120 | tfidf.set_min_document_frequency(2); |
| 121 | DLIB_TEST(tfidf.vocabulary_size() == 4); |
| 122 | tfidf.train(corpus()); |
| 123 | DLIB_TEST(tfidf.vocabulary_size() == 1); |
| 124 | } |
| 125 | |
| 126 | // ---------------------------------------------------------------------------------------- |
| 127 |
no test coverage detected