MCPcopy Create free account
hub / github.com/davisking/dlib / test_term_selection

Function test_term_selection

dlib/test/tfidf.cpp:67–91  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

65// ----------------------------------------------------------------------------------------
66
67 void test_term_selection()
68 {
69 tfidf_transformer<sparse_type> tfidf;
70 tfidf.set_min_document_frequency(2);
71 tfidf.train(corpus());
72 DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({1}));
73 const sparse_type r = tfidf(corpus()[0]);
74 DLIB_TEST(r.size() == 1 && r[0].first == 0 && near(r[0].second, 1));
75
76 // The most frequent terms by total count: term 1 (4), then term 2 (3).
77 tfidf.set_min_document_frequency(1);
78 tfidf.set_max_vocabulary_size(2);
79 tfidf.train(corpus());
80 DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({1,2}));
81
82 // Terms 0 and 3 tie with a total count of 1; the smaller term wins.
83 tfidf.set_max_vocabulary_size(3);
84 tfidf.train(corpus());
85 DLIB_TEST(tfidf.vocabulary() == std::vector<unsigned long>({0,1,2}));
86
87 tfidf.set_max_vocabulary_size(0);
88 tfidf.train(corpus());
89 DLIB_TEST(tfidf.vocabulary_size() == 0);
90 DLIB_TEST(tfidf(corpus()[0]).empty());
91 }
92
93// ----------------------------------------------------------------------------------------
94

Callers 1

perform_testMethod · 0.85

Calls 8

corpusFunction · 0.85
nearFunction · 0.85
vocabulary_sizeMethod · 0.80
trainMethod · 0.45
sizeMethod · 0.45
emptyMethod · 0.45

Tested by

no test coverage detected