| 14 | out_dir.mkdir(parents=True, exist_ok=True) |
| 15 | |
| 16 | def transform_action_data(action_data:dict) -> dict: |
| 17 | |
| 18 | # directly retrieve relevant information from the data. |
| 19 | episode_id:str = action_data["image"].split('/')[-1].split("_")[0] |
| 20 | step_id:str = action_data["image"].split('/')[-1].split("_")[1][:-4] |
| 21 | episode_length:str = action_data["step_length"] |
| 22 | image_path:str = action_data["image"] |
| 23 | instruction:str = action_data["question"] |
| 24 | answer:str = action_data["answer"] |
| 25 | |
| 26 | # Get the picture information (w/h) of the data. They are restored under annotation/*.json. |
| 27 | with open(os.path.join(current_dir, "tmp/GUI-Odyssey/annotations", f"{episode_id}.json"), "r", encoding="utf-8") as f: |
| 28 | data = json.load(f)["device_info"] |
| 29 | image_width:int = data["w"] |
| 30 | image_height:int = data["h"] |
| 31 | |
| 32 | # first intialize the variables. |
| 33 | result_action_text = "" |
| 34 | result_touch_yx = [-1.0,-1.0] |
| 35 | result_lift_yx = [-1.0,-1.0] |
| 36 | duration = 0 |
| 37 | result_action_type:int = 2 |
| 38 | |
| 39 | # case: Click. |
| 40 | if answer.startswith("CLICK"): |
| 41 | result_action_type = 4 |
| 42 | x,y = list(map(float, answer[8:-1].split(","))) |
| 43 | result_touch_yx = [y / 1000, x / 1000] # get the ratio. |
| 44 | result_lift_yx = deepcopy(result_touch_yx) # same point. |
| 45 | |
| 46 | if answer.startswith("SCROLL"): |
| 47 | result_action_type = 4 |
| 48 | |
| 49 | result_touch_yx = [0.5, 0.5] |
| 50 | result_lift_yx = deepcopy(result_touch_yx) |
| 51 | |
| 52 | # Manually create the end point as odyssey didn't give us an exact coordinate. |
| 53 | if answer.endswith("UP"): |
| 54 | result_lift_yx[0] -= 0.1 |
| 55 | |
| 56 | elif answer.endswith("DOWN"): |
| 57 | result_lift_yx[0] += 0.1 |
| 58 | |
| 59 | elif answer.endswith("LEFT"): |
| 60 | result_lift_yx[1] -= 0.1 |
| 61 | |
| 62 | else: |
| 63 | result_lift_yx[1] += 0.1 |
| 64 | |
| 65 | if answer.startswith("LONG_PRESS"): |
| 66 | result_action_type = 0 |
| 67 | |
| 68 | x,y = list(map(float, answer[13:-1].split(","))) |
| 69 | result_touch_yx = [y / 1000, x / 1000] |
| 70 | |
| 71 | result_lift_yx = result_lift_yx = deepcopy(result_touch_yx) |
| 72 | |
| 73 | if answer.startswith("TYPE"): |