| 260 | } |
| 261 | |
| 262 | char32_t DefaultTokenizer::peekCodePoint() { |
| 263 | if (index_ >= path_.size()) { |
| 264 | return 0; |
| 265 | } |
| 266 | const unsigned char* p = |
| 267 | reinterpret_cast<const unsigned char*>(path_.data() + index_); |
| 268 | const unsigned char* e = |
| 269 | reinterpret_cast<const unsigned char*>(path_.data() + path_.size()); |
| 270 | const unsigned char* orig = p; |
| 271 | char32_t cp = folly::utf8ToCodePoint(p, e, /*skipOnError=*/false); |
| 272 | if (p == orig) { |
| 273 | invalidSubfieldPath(); // Invalid UTF-8 |
| 274 | } |
| 275 | return cp; |
| 276 | } |
| 277 | |
| 278 | void DefaultTokenizer::consumeCodePoint() { |
| 279 | if (index_ >= path_.size()) |