Function tokenize

src/tokenizer.rs:126–328

Ref: https://wicg.github.io/urlpattern/#tokenize

(
  input: &str,
  policy: TokenizePolicy,
)

// Ref: https://wicg.github.io/urlpattern/#tokenize
pub fn tokenize(
  input: &str,
  policy: TokenizePolicy,
) -> Result<Vec<Token>, Error> {
  let mut tokenizer = Tokenizer {
    input,
    policy,
    token_list: vec![],
    index: 0,
    next_index: 0,
    code_point: None,
  };

  while tokenizer.index < tokenizer.input.len() {
    tokenizer.seek_and_get_next_codepoint(tokenizer.index);

    if tokenizer.code_point == Some('*') {
      tokenizer.add_token_with_default_pos_and_len(TokenType::Asterisk);
      continue;
    }
    if matches!(tokenizer.code_point, Some('+') | Some('?')) {
      tokenizer.add_token_with_default_pos_and_len(TokenType::OtherModifier);
      continue;
    }
    if tokenizer.code_point == Some('\\') {
      if tokenizer.index == (tokenizer.input.len() - 1) {
        tokenizer.process_tokenizing_error(
          tokenizer.next_index,
          tokenizer.index,
          TokenizerError::IncompleteEscapeCode,
        )?;
        continue;
      }
      let escaped_index = tokenizer.next_index;
      tokenizer.get_next_codepoint();
      tokenizer.add_token_with_default_len(
        TokenType::EscapedChar,
        tokenizer.next_index,
        escaped_index,
      );
      continue;
    }
    if tokenizer.code_point == Some('\n')
      || tokenizer.code_point == Some('\r')
      || tokenizer.code_point == Some('\t')
    {
      // ignore newline, carriage return and tab
      tokenizer.index = tokenizer.next_index;
      continue;
    }
    if tokenizer.code_point == Some('{') {
      tokenizer.add_token_with_default_pos_and_len(TokenType::Open);
      continue;
    }
    if tokenizer.code_point == Some('}') {
      tokenizer.add_token_with_default_pos_and_len(TokenType::Close);
      continue;
    }