(env, trajectory, render_helper, traces, state_info, agent, args, intent, meta_data, target_action)
| 35 | return act_nodes |
| 36 | |
| 37 | def step_once(env, trajectory, render_helper, traces, state_info, agent, args, intent, meta_data, target_action): |
| 38 | obs_info = state_info["info"]["observation_metadata"]["text"] |
| 39 | images = state_info["info"]["images"] |
| 40 | dom_info, nodes_info = obs_info["dom_info"], obs_info["obs_nodes_info"] |
| 41 | dom_tree = dom_info["dom_tree"] |
| 42 | raw_html = dom_info["raw_html"] |
| 43 | |
| 44 | if target_action.count('#Type#') > 0 and target_action.endswith('\\n'): |
| 45 | target_action = target_action[:-2] + '\n' |
| 46 | |
| 47 | prompt, action = agent.check_action( |
| 48 | trajectory, intent, meta_data, target_action |
| 49 | ) |
| 50 | |
| 51 | print('[prompt] ', prompt) |
| 52 | print('[action] ', action) |
| 53 | |
| 54 | # our_dom_tree = copy.deepcopy(dom_tree) |
| 55 | # for elem in our_dom_tree: |
| 56 | # elem["union_bound"] = elem["union_bound"].tolist() |
| 57 | |
| 58 | myaction = copy.deepcopy(action) |
| 59 | myaction["coords"] = myaction["coords"].tolist() |
| 60 | |
| 61 | need_to_keep = action['action_type'] != ActionTypes.NONE or target_action.count("#Record#") > 0 |
| 62 | |
| 63 | trajectory.append(action) |
| 64 | |
| 65 | action_str = get_action_description( |
| 66 | action, |
| 67 | state_info["info"]["observation_metadata"], |
| 68 | action_set_tag=args.action_set_tag, |
| 69 | prompt_constructor=agent.prompt_constructor |
| 70 | if isinstance(agent, PromptAgent) |
| 71 | else None, |
| 72 | ) |
| 73 | |
| 74 | if need_to_keep: |
| 75 | user_action = action_str.split(' #HTML Segment')[0] |
| 76 | traces.append({ |
| 77 | 'source': prompt, |
| 78 | 'target': f'{real_action}', |
| 79 | 'extra_data': { |
| 80 | 'element_id': action.get('element_id', ''), |
| 81 | 'dom_tree': dom_tree, |
| 82 | 'raw_html': raw_html, |
| 83 | 'nodes_info': nodes_info, |
| 84 | 'raw_action': myaction, |
| 85 | 'images': images, |
| 86 | }, |
| 87 | }) |
| 88 | |
| 89 | render_helper.render( |
| 90 | action, state_info, meta_data, args.render_screenshot |
| 91 | ) |
| 92 | |
| 93 | if need_to_keep: |
| 94 | meta_data["action_history"].append(action_str) |
no test coverage detected