| 21 | typename tok |
| 22 | > |
| 23 | void tokenizer_kernel_test ( |
| 24 | ) |
| 25 | /*! |
| 26 | requires |
| 27 | - tok is an implementation of tokenizer_kernel_abstract.h |
| 28 | ensures |
| 29 | - runs tests on tok for compliance with the specs |
| 30 | !*/ |
| 31 | { |
| 32 | |
| 33 | print_spinner(); |
| 34 | |
| 35 | tok test; |
| 36 | |
| 37 | DLIB_TEST(test.numbers() == "0123456789"); |
| 38 | DLIB_TEST(test.uppercase_letters() == "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); |
| 39 | DLIB_TEST(test.lowercase_letters() == "abcdefghijklmnopqrstuvwxyz"); |
| 40 | |
| 41 | DLIB_TEST_MSG(test.get_identifier_body() == "_" + test.lowercase_letters() + |
| 42 | test.uppercase_letters() + test.numbers(),""); |
| 43 | DLIB_TEST_MSG(test.get_identifier_head() == "_" + test.lowercase_letters() + |
| 44 | test.uppercase_letters(),""); |
| 45 | |
| 46 | DLIB_TEST(test.stream_is_set() == false); |
| 47 | test.clear(); |
| 48 | DLIB_TEST(test.stream_is_set() == false); |
| 49 | |
| 50 | DLIB_TEST_MSG(test.get_identifier_body() == "_" + test.lowercase_letters() + |
| 51 | test.uppercase_letters() + test.numbers(),""); |
| 52 | DLIB_TEST_MSG(test.get_identifier_head() == "_" + test.lowercase_letters() + |
| 53 | test.uppercase_letters(),""); |
| 54 | |
| 55 | tok test2; |
| 56 | |
| 57 | ostringstream sout; |
| 58 | istringstream sin; |
| 59 | test2.set_stream(sin); |
| 60 | |
| 61 | DLIB_TEST(test2.stream_is_set()); |
| 62 | DLIB_TEST(&test2.get_stream() == &sin); |
| 63 | |
| 64 | int type; |
| 65 | string token; |
| 66 | |
| 67 | test2.get_token(type,token); |
| 68 | DLIB_TEST(type == tok::END_OF_FILE); |
| 69 | test2.get_token(type,token); |
| 70 | DLIB_TEST(type == tok::END_OF_FILE); |
| 71 | test2.get_token(type,token); |
| 72 | DLIB_TEST(type == tok::END_OF_FILE); |
| 73 | |
| 74 | |
| 75 | sin.clear(); |
| 76 | sin.str(" The cat 123asdf1234 ._ \n test."); |
| 77 | |
| 78 | test2.get_token(type,token); |
| 79 | DLIB_TEST(type == tok::WHITE_SPACE); |
| 80 | DLIB_TEST(token == " "); |
nothing calls this directly
no test coverage detected