MCPcopy Create free account
hub / github.com/davisking/dlib / test_ngram_lists

Function test_ngram_lists

dlib/test/feature_hashing.cpp:136–162  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

134// ----------------------------------------------------------------------------------------
135
136 void test_ngram_lists()
137 {
138 const std::vector<std::string> tokens = {"a", "b", "a", "b"};
139 DLIB_TEST(word_ngrams(tokens, 1, 2) ==
140 std::vector<std::string>({"a", "b", "a", "b", "a b", "b a", "a b"}));
141 DLIB_TEST(word_ngrams(tokens, 3, 3) == std::vector<std::string>({"a b a", "b a b"}));
142 DLIB_TEST(word_ngrams(tokens, 5, 6).empty());
143
144 DLIB_TEST(char_ngrams(std::vector<std::string>{"abc", "a"}, 3, 4) ==
145 std::vector<std::string>({" ab", "abc", "bc ", " abc", "abc ", " a "}));
146 DLIB_TEST(char_ngrams(std::vector<std::string>{"a"}, 4, 5) == std::vector<std::string>({" a "}));
147 DLIB_TEST(char_ngrams(std::vector<std::string>(), 3, 5).empty());
148
149 const std::vector<std::string> words = {"to", "be", "or", "not", "to", "be"};
150 const std::vector<std::string_view> views(words.begin(), words.end());
151 DLIB_TEST(word_ngrams(views, 1, 3) == word_ngrams(words, 1, 3));
152 DLIB_TEST(char_ngrams(views, 2, 4) == char_ngrams(words, 2, 4));
153
154 // The hashing functions count exactly these n-grams.
155 const auto text = split("the quick brown fox jumps over the lazy dog and the quick cat");
156 for (const unsigned long num_buckets : {1ul, 1000ul, 1ul << 20})
157 {
158 DLIB_TEST(hash_word_ngrams(text, 1, 3, num_buckets) == expected(word_ngrams(text, 1, 3), num_buckets));
159 DLIB_TEST(hash_char_ngrams(text, 2, 5, num_buckets) == expected(char_ngrams(text, 2, 5), num_buckets));
160 DLIB_TEST(hash_char_ngrams(text, 6, 8, num_buckets) == expected(char_ngrams(text, 6, 8), num_buckets));
161 }
162 }
163
164// ----------------------------------------------------------------------------------------
165

Callers 1

perform_testMethod · 0.85

Calls 9

word_ngramsFunction · 0.85
char_ngramsFunction · 0.85
hash_word_ngramsFunction · 0.85
expectedFunction · 0.85
hash_char_ngramsFunction · 0.85
splitFunction · 0.50
emptyMethod · 0.45
beginMethod · 0.45
endMethod · 0.45

Tested by

no test coverage detected