MCPcopy Create free account
hub / github.com/davisking/dlib / test_weighting_options

Function test_weighting_options

dlib/test/tfidf.cpp:95–124  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

93// ----------------------------------------------------------------------------------------
94
95 void test_weighting_options()
96 {
97 tfidf_transformer<sparse_type> tfidf;
98 tfidf.normalize_to_unit_length(false);
99 tfidf.train(corpus());
100
101 sparse_type r = tfidf(corpus()[1]);
102 DLIB_TEST(r.size() == 2);
103 DLIB_TEST(r[0].first == 1 && near(r[0].second, 1*1.0));
104 DLIB_TEST(r[1].first == 2 && near(r[1].second, 3*(std::log(2.0)+1)));
105
106 // Weighting options apply to operator() right away.
107 tfidf.use_sublinear_tf(true);
108 r = tfidf(corpus()[1]);
109 DLIB_TEST(near(r[0].second, (1 + std::log(1.0))*1));
110 DLIB_TEST(near(r[1].second, (1 + std::log(3.0))*(std::log(2.0)+1)));
111
112 // idf smoothing applies on the next train().
113 tfidf.use_smooth_idf(false);
114 DLIB_TEST(near(tfidf.idf()[0], std::log(2.0) + 1));
115 tfidf.train(corpus());
116 DLIB_TEST(near(tfidf.idf()[0], std::log(3.0) + 1));
117 DLIB_TEST(near(tfidf.idf()[1], 1));
118
119 // So does term selection.
120 tfidf.set_min_document_frequency(2);
121 DLIB_TEST(tfidf.vocabulary_size() == 4);
122 tfidf.train(corpus());
123 DLIB_TEST(tfidf.vocabulary_size() == 1);
124 }
125
126// ----------------------------------------------------------------------------------------
127

Callers 1

perform_testMethod · 0.85

Calls 10

corpusFunction · 0.85
nearFunction · 0.85
logFunction · 0.85
use_sublinear_tfMethod · 0.80
use_smooth_idfMethod · 0.80
vocabulary_sizeMethod · 0.80
trainMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected