Method add_token

test-app/runtime/src/main/cpp/ada/ada.h:9817–9833

}

inline void Tokenizer::add_token(token_type type, size_t next_position,
                                 size_t value_position, size_t value_length) {
  ada_log("Tokenizer::add_token called with type=", to_string(type),
          " next_position=", next_position, " value_position=", value_position);
  ADA_ASSERT_TRUE(next_position >= value_position);

  // Let token be a new token.
  // Set token's type to type.
  // Set token's index to tokenizer's index.
  // Set token's value to the code point substring from value position with
  // length value length within tokenizer's input.
  // Append token to the back of tokenizer's token list.
  token_list.emplace_back(type, index,
                          input.substr(value_position, value_length));
  // Set tokenizer's index to next position.
  index = next_position;
}

inline void Tokenizer::add_token_with_default_length(token_type type,
                                                     size_t next_position,