| 9833 | } |
| 9834 | |
| 9835 | inline void Tokenizer::add_token_with_default_length(token_type type, |
| 9836 | size_t next_position, |
| 9837 | size_t value_position) { |
| 9838 | // Let computed length be next position - value position. |
| 9839 | auto computed_length = next_position - value_position; |
| 9840 | // Run add a token given tokenizer, type, next position, value position, and |
| 9841 | // computed length. |
| 9842 | add_token(type, next_position, value_position, computed_length); |
| 9843 | } |
| 9844 | |
| 9845 | inline void Tokenizer::add_token_with_defaults(token_type type) { |
| 9846 | ada_log("Tokenizer::add_token_with_defaults called with type=", |