| 134 | // ---------------------------------------------------------------------------------------- |
| 135 | |
| 136 | void test_ngram_lists() |
| 137 | { |
| 138 | const std::vector<std::string> tokens = {"a", "b", "a", "b"}; |
| 139 | DLIB_TEST(word_ngrams(tokens, 1, 2) == |
| 140 | std::vector<std::string>({"a", "b", "a", "b", "a b", "b a", "a b"})); |
| 141 | DLIB_TEST(word_ngrams(tokens, 3, 3) == std::vector<std::string>({"a b a", "b a b"})); |
| 142 | DLIB_TEST(word_ngrams(tokens, 5, 6).empty()); |
| 143 | |
| 144 | DLIB_TEST(char_ngrams(std::vector<std::string>{"abc", "a"}, 3, 4) == |
| 145 | std::vector<std::string>({" ab", "abc", "bc ", " abc", "abc ", " a "})); |
| 146 | DLIB_TEST(char_ngrams(std::vector<std::string>{"a"}, 4, 5) == std::vector<std::string>({" a "})); |
| 147 | DLIB_TEST(char_ngrams(std::vector<std::string>(), 3, 5).empty()); |
| 148 | |
| 149 | const std::vector<std::string> words = {"to", "be", "or", "not", "to", "be"}; |
| 150 | const std::vector<std::string_view> views(words.begin(), words.end()); |
| 151 | DLIB_TEST(word_ngrams(views, 1, 3) == word_ngrams(words, 1, 3)); |
| 152 | DLIB_TEST(char_ngrams(views, 2, 4) == char_ngrams(words, 2, 4)); |
| 153 | |
| 154 | // The hashing functions count exactly these n-grams. |
| 155 | const auto text = split("the quick brown fox jumps over the lazy dog and the quick cat"); |
| 156 | for (const unsigned long num_buckets : {1ul, 1000ul, 1ul << 20}) |
| 157 | { |
| 158 | DLIB_TEST(hash_word_ngrams(text, 1, 3, num_buckets) == expected(word_ngrams(text, 1, 3), num_buckets)); |
| 159 | DLIB_TEST(hash_char_ngrams(text, 2, 5, num_buckets) == expected(char_ngrams(text, 2, 5), num_buckets)); |
| 160 | DLIB_TEST(hash_char_ngrams(text, 6, 8, num_buckets) == expected(char_ngrams(text, 6, 8), num_buckets)); |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // ---------------------------------------------------------------------------------------- |
| 165 |
no test coverage detected