Convert the ui-tars action string to the minicpm schema format Args: action_str (str): like "click(start_box='<|box_start|>(558,925)<|box_end|>')" Returns: dict: new format action dictionary
(action_str)
| 283 | return episode |
| 284 | |
| 285 | def uitars2minicpm(action_str): |
| 286 | """ |
| 287 | Convert the ui-tars action string to the minicpm schema format |
| 288 | |
| 289 | Args: |
| 290 | action_str (str): like "click(start_box='<|box_start|>(558,925)<|box_end|>')" |
| 291 | |
| 292 | Returns: |
| 293 | dict: new format action dictionary |
| 294 | """ |
| 295 | result = {"STATUS": "continue"} |
| 296 | |
| 297 | # auxiliary function to extract coordinates |
| 298 | def extract_coords(s): |
| 299 | # directly find and extract the coordinates in the parentheses |
| 300 | first_bracket = s.find("(") |
| 301 | start = s.find("(", first_bracket + 1) |
| 302 | end = s.find(")") |
| 303 | if start != -1 and end != -1: |
| 304 | coords_str = s[start+1:end].strip() # extract the content in (x,y) |
| 305 | x, y = coords_str.split(",") |
| 306 | return [int(x), int(y)] |
| 307 | raise ValueError(f"Cannot find coordinates in the string: {s}") |
| 308 | |
| 309 | if "click(" in action_str: |
| 310 | result["POINT"] = extract_coords(action_str) |
| 311 | |
| 312 | elif "long_press(" in action_str: |
| 313 | result["POINT"] = extract_coords(action_str) |
| 314 | if "time='" in action_str: |
| 315 | time = action_str.split("time='")[1].split("'")[0] |
| 316 | result["duration"] = int(time) if time else 1000 |
| 317 | |
| 318 | elif "type(" in action_str: |
| 319 | content = action_str.split("content='")[1].split("'")[0] |
| 320 | result["TYPE"] = content |
| 321 | |
| 322 | elif "scroll(" in action_str: |
| 323 | direction = action_str.split("direction='")[1].split("'")[0] |
| 324 | result["POINT"] = [500, 500] # screen center point |
| 325 | #need reverse direction |
| 326 | if direction == "down": |
| 327 | direction = "up" |
| 328 | elif direction == "up": |
| 329 | direction = "down" |
| 330 | elif direction == "right": |
| 331 | direction = "left" |
| 332 | elif direction == "left": |
| 333 | direction = "right" |
| 334 | result["to"] = direction |
| 335 | elif "press_back()" in action_str: |
| 336 | result["PRESS"] = "BACK" |
| 337 | |
| 338 | elif "press_home()" in action_str: |
| 339 | result["PRESS"] = "HOME" |
| 340 | |
| 341 | elif "wait()" in action_str: |
| 342 | result["duration"] = 200 |
no test coverage detected