| 50 | // ---------------------------------------------------------------------------------------- |
| 51 | |
| 52 | void test_char_ngrams() |
| 53 | { |
| 54 | const unsigned long num_buckets = 1 << 20; |
| 55 | |
| 56 | // " ab " has two 3-grams, is itself the only 4-gram, and is too short for 5. |
| 57 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"ab"}, 3, 5, num_buckets) == |
| 58 | expected({" ab", "ab ", " ab "}, num_buckets)); |
| 59 | |
| 60 | // " a " is shorter than min_n, so it counts once, whole. |
| 61 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"a"}, 4, 5, num_buckets) == |
| 62 | expected({" a "}, num_buckets)); |
| 63 | |
| 64 | // N-grams stay inside their token. |
| 65 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abc", "de"}, 3, 3, num_buckets) == |
| 66 | expected({" ab", "abc", "bc ", " de", "de "}, num_buckets)); |
| 67 | |
| 68 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abcd"}, 2, 3, num_buckets) == |
| 69 | expected({" a", "ab", "bc", "cd", "d ", " ab", "abc", "bcd", "cd "}, num_buckets)); |
| 70 | |
| 71 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>(), 3, 5, num_buckets).empty()); |
| 72 | |
| 73 | // UTF-8 e-acute has two bytes, so its padded form has two byte trigrams. |
| 74 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 3, 3, num_buckets) == |
| 75 | expected({" \xc3\xa9", "\xc3\xa9 "}, num_buckets)); |
| 76 | DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 1, 1, num_buckets) == |
| 77 | expected({" ", "\xc3", "\xa9", " "}, num_buckets)); |
| 78 | } |
| 79 | |
| 80 | // ---------------------------------------------------------------------------------------- |
| 81 |
no test coverage detected