| 392 | std::vector<bool> isSpace; |
| 393 | |
| 394 | explicit BasicTokenizer(bool lower_case) : do_lower_case(lower_case) { |
| 395 | char_map.assign(256, 0); |
| 396 | for (int i = 0; i < 256; ++i) { |
| 397 | if (i == 0) { |
| 398 | char_map[i] = 0; |
| 399 | } else if (i < 128) { |
| 400 | char_map[i] = std::isspace(i) ? ' ' : i; |
| 401 | } else { |
| 402 | char_map[i] = i; |
| 403 | } |
| 404 | } |
| 405 | isSpace.assign(256, false); |
| 406 | isSpace[' '] = true; |
| 407 | isSpace['\t'] = true; |
| 408 | isSpace['\n'] = true; |
| 409 | isSpace['\v'] = true; |
| 410 | isSpace['\f'] = true; |
| 411 | isSpace['\r'] = true; |
| 412 | } |
| 413 | |
| 414 | |
| 415 | [[nodiscard]] STRING_LIST tokenize(const std::string &text) const { |
nothing calls this directly
no outgoing calls
no test coverage detected