For basic text extraction, choose 'whiteSpace' to be ASCII values 0x00-0x20,0x7F-0xFF in GetTokens(...).
| 96 | // For basic text extraction, choose 'whiteSpace' to be ASCII values |
| 97 | // 0x00-0x20,0x7F-0xFF in GetTokens(...). |
| 98 | inline void GetTextTokens(std::string const& input, |
| 99 | std::vector<std::string>& tokens) |
| 100 | { |
| 101 | static std::string const whiteSpace = [] |
| 102 | { |
| 103 | std::string temp; |
| 104 | for (int32_t i = 0; i <= 32; ++i) |
| 105 | { |
| 106 | temp += char(i); |
| 107 | } |
| 108 | for (int32_t i = 127; i < 255; ++i) |
| 109 | { |
| 110 | temp += char(i); |
| 111 | } |
| 112 | return temp; |
| 113 | } |
| 114 | (); |
| 115 | |
| 116 | GetTokens(input, whiteSpace, tokens); |
| 117 | } |
| 118 | |
| 119 | // For advanced text extraction, choose 'whiteSpace' to be ASCII values |
| 120 | // 0x00-0x20,0x7F in GetTokens(...). Any special characters for ASCII |
nothing calls this directly
no test coverage detected