| 9851 | } |
| 9852 | |
| 9853 | inline ada_warn_unused std::optional<errors> |
| 9854 | Tokenizer::process_tokenizing_error(size_t next_position, |
| 9855 | size_t value_position) { |
| 9856 | // If tokenizer's policy is "strict", then throw a TypeError. |
| 9857 | if (policy == token_policy::strict) { |
| 9858 | ada_log("process_tokenizing_error failed with next_position=", |
| 9859 | next_position, " value_position=", value_position); |
| 9860 | return errors::type_error; |
| 9861 | } |
| 9862 | // Assert: tokenizer's policy is "lenient". |
| 9863 | ADA_ASSERT_TRUE(policy == token_policy::lenient); |
| 9864 | // Run add a token with default length given tokenizer, "invalid-char", next |
| 9865 | // position, and value position. |
| 9866 | add_token_with_default_length(token_type::INVALID_CHAR, next_position, |
| 9867 | value_position); |
| 9868 | return std::nullopt; |
| 9869 | } |
| 9870 | |
| 9871 | template <url_pattern_encoding_callback F> |
| 9872 | token* url_pattern_parser<F>::try_consume_modifier_token() { |