| 9806 | } |
| 9807 | |
| 9808 | constexpr void Tokenizer::seek_and_get_next_code_point(size_t new_index) { |
| 9809 | ada_log("Tokenizer::seek_and_get_next_code_point called with new_index=", |
| 9810 | new_index); |
| 9811 | // Set tokenizer's next index to index. |
| 9812 | next_index = new_index; |
| 9813 | // Run get the next code point given tokenizer. |
| 9814 | get_next_code_point(); |
| 9815 | } |
| 9816 | |
| 9817 | inline void Tokenizer::add_token(token_type type, size_t next_position, |
| 9818 | size_t value_position, size_t value_length) { |