MCPcopy Create free account
hub / github.com/davisking/dlib / test_char_ngrams

Function test_char_ngrams

dlib/test/feature_hashing.cpp:52–78  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

50// ----------------------------------------------------------------------------------------
51
52 void test_char_ngrams()
53 {
54 const unsigned long num_buckets = 1 << 20;
55
56 // " ab " has two 3-grams, is itself the only 4-gram, and is too short for 5.
57 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"ab"}, 3, 5, num_buckets) ==
58 expected({" ab", "ab ", " ab "}, num_buckets));
59
60 // " a " is shorter than min_n, so it counts once, whole.
61 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"a"}, 4, 5, num_buckets) ==
62 expected({" a "}, num_buckets));
63
64 // N-grams stay inside their token.
65 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abc", "de"}, 3, 3, num_buckets) ==
66 expected({" ab", "abc", "bc ", " de", "de "}, num_buckets));
67
68 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"abcd"}, 2, 3, num_buckets) ==
69 expected({" a", "ab", "bc", "cd", "d ", " ab", "abc", "bcd", "cd "}, num_buckets));
70
71 DLIB_TEST(hash_char_ngrams(std::vector<std::string>(), 3, 5, num_buckets).empty());
72
73 // UTF-8 e-acute has two bytes, so its padded form has two byte trigrams.
74 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 3, 3, num_buckets) ==
75 expected({" \xc3\xa9", "\xc3\xa9 "}, num_buckets));
76 DLIB_TEST(hash_char_ngrams(std::vector<std::string>{"\xc3\xa9"}, 1, 1, num_buckets) ==
77 expected({" ", "\xc3", "\xa9", " "}, num_buckets));
78 }
79
80// ----------------------------------------------------------------------------------------
81

Callers 1

perform_testMethod · 0.85

Calls 3

hash_char_ngramsFunction · 0.85
expectedFunction · 0.85
emptyMethod · 0.45

Tested by

no test coverage detected