MCPcopy Create free account
hub / github.com/davisking/dlib / tokenizer_kernel_test

Function tokenizer_kernel_test

dlib/test/tokenizer.cpp:23–352  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

21 typename tok
22 >
23 void tokenizer_kernel_test (
24 )
25 /*!
26 requires
27 - tok is an implementation of tokenizer_kernel_abstract.h
28 ensures
29 - runs tests on tok for compliance with the specs
30 !*/
31 {
32
33 print_spinner();
34
35 tok test;
36
37 DLIB_TEST(test.numbers() == "0123456789");
38 DLIB_TEST(test.uppercase_letters() == "ABCDEFGHIJKLMNOPQRSTUVWXYZ");
39 DLIB_TEST(test.lowercase_letters() == "abcdefghijklmnopqrstuvwxyz");
40
41 DLIB_TEST_MSG(test.get_identifier_body() == "_" + test.lowercase_letters() +
42 test.uppercase_letters() + test.numbers(),"");
43 DLIB_TEST_MSG(test.get_identifier_head() == "_" + test.lowercase_letters() +
44 test.uppercase_letters(),"");
45
46 DLIB_TEST(test.stream_is_set() == false);
47 test.clear();
48 DLIB_TEST(test.stream_is_set() == false);
49
50 DLIB_TEST_MSG(test.get_identifier_body() == "_" + test.lowercase_letters() +
51 test.uppercase_letters() + test.numbers(),"");
52 DLIB_TEST_MSG(test.get_identifier_head() == "_" + test.lowercase_letters() +
53 test.uppercase_letters(),"");
54
55 tok test2;
56
57 ostringstream sout;
58 istringstream sin;
59 test2.set_stream(sin);
60
61 DLIB_TEST(test2.stream_is_set());
62 DLIB_TEST(&test2.get_stream() == &sin);
63
64 int type;
65 string token;
66
67 test2.get_token(type,token);
68 DLIB_TEST(type == tok::END_OF_FILE);
69 test2.get_token(type,token);
70 DLIB_TEST(type == tok::END_OF_FILE);
71 test2.get_token(type,token);
72 DLIB_TEST(type == tok::END_OF_FILE);
73
74
75 sin.clear();
76 sin.str(" The cat 123asdf1234 ._ \n test.");
77
78 test2.get_token(type,token);
79 DLIB_TEST(type == tok::WHITE_SPACE);
80 DLIB_TEST(token == " ");

Callers

nothing calls this directly

Calls 5

print_spinnerFunction · 0.85
swapFunction · 0.70
clearMethod · 0.45
strMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected