| 276 | } |
| 277 | |
| 278 | void DefaultTokenizer::consumeCodePoint() { |
| 279 | if (index_ >= path_.size()) |
| 280 | return; |
| 281 | const unsigned char* p = |
| 282 | reinterpret_cast<const unsigned char*>(path_.data() + index_); |
| 283 | const unsigned char* e = |
| 284 | reinterpret_cast<const unsigned char*>(path_.data() + path_.size()); |
| 285 | const unsigned char* orig = p; |
| 286 | folly::utf8ToCodePoint(p, e, /*skipOnError=*/false); |
| 287 | if (p == orig) { |
| 288 | invalidSubfieldPath(); |
| 289 | } |
| 290 | index_ += (p - orig); |
| 291 | } |
| 292 | |
| 293 | void DefaultTokenizer::consumeCurrentCodePoint() { |
| 294 | if (index_ >= path_.size()) { |