| 151 | } |
| 152 | |
| 153 | std::vector<absl::string_view> SplitIntoWords(absl::string_view text, |
| 154 | bool treat_ws_as_suffix, |
| 155 | bool allow_ws_only_pieces) { |
| 156 | const char *begin = text.data(); |
| 157 | const char *end = text.data() + text.size(); |
| 158 | |
| 159 | // Space symbol (U+2581) |
| 160 | const absl::string_view kSpaceSymbol = "\xe2\x96\x81"; |
| 161 | bool in_ws_sequence = false; |
| 162 | |
| 163 | std::vector<absl::string_view> result; |
| 164 | if (treat_ws_as_suffix) { // put ws tokens at the end of non-ws sequences. |
| 165 | if (begin < end) result.emplace_back(begin, 0); |
| 166 | while (begin < end) { |
| 167 | const int mblen = |
| 168 | std::min<int>(string_util::OneCharLen(begin), end - begin); |
| 169 | const bool is_ws = absl::string_view(begin, mblen) == kSpaceSymbol; |
| 170 | |
| 171 | if (is_ws) { // keep track of sequences consecutive ws tokens. |
| 172 | in_ws_sequence = true; |
| 173 | } else if (in_ws_sequence) { |
| 174 | if (allow_ws_only_pieces) result.emplace_back(begin, 0); |
| 175 | |
| 176 | in_ws_sequence = false; |
| 177 | } |
| 178 | |
| 179 | result.back() = |
| 180 | absl::string_view(result.back().data(), result.back().size() + mblen); |
| 181 | begin += mblen; |
| 182 | |
| 183 | if (begin < end && is_ws && !allow_ws_only_pieces) |
| 184 | result.emplace_back(begin, 0); |
| 185 | } |
| 186 | } else { |
| 187 | while (begin < end) { |
| 188 | const int mblen = |
| 189 | std::min<int>(string_util::OneCharLen(begin), end - begin); |
| 190 | bool is_ws = absl::string_view(begin, mblen) == kSpaceSymbol; |
| 191 | |
| 192 | // if is whitespace (and not in sequence if allow_ws_only_pieces is True) |
| 193 | if (begin == text.data() || |
| 194 | (is_ws && (!in_ws_sequence || !allow_ws_only_pieces))) { |
| 195 | result.emplace_back(begin, 0); // add empty string piece. |
| 196 | in_ws_sequence = true; |
| 197 | } |
| 198 | |
| 199 | if (in_ws_sequence && !is_ws) in_ws_sequence = false; |
| 200 | |
| 201 | result.back() = |
| 202 | absl::string_view(result.back().data(), result.back().size() + mblen); |
| 203 | begin += mblen; |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | return result; |
| 208 | } |
| 209 | |
| 210 | std::string ByteToPiece(unsigned char c) { |