Helper class to render text and image observations and meta data in the trajectory
| 114 | |
| 115 | |
| 116 | class RenderHelper(object): |
| 117 | """Helper class to render text and image observations and meta data in the trajectory""" |
| 118 | |
| 119 | def __init__( |
| 120 | self, config_file: str, result_dir: str, action_set_tag: str |
| 121 | ) -> None: |
| 122 | with open(config_file, "r") as f: |
| 123 | _config = json.load(f) |
| 124 | _config_str = "" |
| 125 | for k, v in _config.items(): |
| 126 | _config_str += f"{k}: {v}\n" |
| 127 | _config_str = f"<pre>{_config_str}</pre>\n" |
| 128 | task_id = _config["task_id"] |
| 129 | |
| 130 | self.action_set_tag = action_set_tag |
| 131 | |
| 132 | self.render_file = open( |
| 133 | Path(result_dir) / f"render_{task_id}.html", "a+" |
| 134 | ) |
| 135 | self.render_file.truncate(0) |
| 136 | # write init template |
| 137 | self.render_file.write(HTML_TEMPLATE.format(body=f"{_config_str}")) |
| 138 | self.render_file.read() |
| 139 | self.render_file.flush() |
| 140 | |
| 141 | def render( |
| 142 | self, |
| 143 | action: Action, |
| 144 | state_info: StateInfo, |
| 145 | meta_data: dict[str, Any], |
| 146 | render_screenshot: bool = False, |
| 147 | ) -> None: |
| 148 | """Render the trajectory""" |
| 149 | # text observation |
| 150 | observation = state_info["observation"] |
| 151 | text_obs = observation["text"] |
| 152 | info = state_info["info"] |
| 153 | new_content = f"<h2>New Page</h2>\n" |
| 154 | new_content += f"<h3 class='url'><a href={state_info['info']['page'].url}>URL: {state_info['info']['page'].url}</a></h3>\n" |
| 155 | new_content += f"<div class='state_obv'><pre>{text_obs}</pre><div>\n" |
| 156 | |
| 157 | if render_screenshot: |
| 158 | # image observation |
| 159 | img_obs = observation["image"] |
| 160 | image = Image.fromarray(img_obs) |
| 161 | byte_io = io.BytesIO() |
| 162 | image.save(byte_io, format="PNG") |
| 163 | byte_io.seek(0) |
| 164 | image_bytes = base64.b64encode(byte_io.read()) |
| 165 | image_str = image_bytes.decode("utf-8") |
| 166 | new_content += f"<img src='data:image/png;base64,{image_str}' style='width:50vw; height:auto;'/>\n" |
| 167 | |
| 168 | # meta data |
| 169 | new_content += f"<div class='prev_action' style='background-color:pink'>{meta_data['action_history'][-1]}</div>\n" |
| 170 | |
| 171 | # action |
| 172 | action_str = get_render_action( |
| 173 | action, |