(example)
| 20 | |
| 21 | |
| 22 | def extract_gt_action(example): |
| 23 | ex_action_type = example['result_action_type'] |
| 24 | |
| 25 | if ex_action_type == ActionType.DUAL_POINT: |
| 26 | lift_yx = json.loads(example['result_lift_yx']) |
| 27 | touch_yx = json.loads(example['result_touch_yx']) |
| 28 | if is_tap_action(np.array(touch_yx), np.array(lift_yx)): |
| 29 | action_type = 'CLICK' |
| 30 | w, h = example['image_width'], example['image_height'] |
| 31 | assert w and h, "Invalid image size" |
| 32 | click_y, click_x = round(lift_yx[0] * h), round(lift_yx[1] * w) # Note: image size range, not 0-1000 |
| 33 | action = (click_y, click_x) |
| 34 | else: |
| 35 | action_type = 'SCROLL' |
| 36 | v_change = abs(touch_yx[0] - lift_yx[0]) |
| 37 | h_change = abs(lift_yx[1] - touch_yx[1]) |
| 38 | is_scroll_up = lift_yx[0] < touch_yx[0] # touch is lower |
| 39 | is_scroll_left = lift_yx[1] < touch_yx[1] # touch is bigger |
| 40 | if v_change >= 0.9*h_change: # vertical |
| 41 | action = "scroll up" if is_scroll_up else "scroll down" |
| 42 | else: # horizonal |
| 43 | action = "scroll left" if is_scroll_left else "scroll right" |
| 44 | elif ex_action_type in ( |
| 45 | ActionType.PRESS_BACK, |
| 46 | ActionType.PRESS_HOME, |
| 47 | ): |
| 48 | button = ActionType(ex_action_type).name.split('_')[1].lower() |
| 49 | action = f'press the {button} button' |
| 50 | action_type = f'PRESS {button}'.upper() |
| 51 | elif ex_action_type == ActionType.PRESS_ENTER: |
| 52 | action = "press enter" |
| 53 | action_type = 'PRESS ENTER' |
| 54 | elif ex_action_type == ActionType.TYPE: |
| 55 | action_text = example['result_action_text'] |
| 56 | action = f'input text "{action_text}"' |
| 57 | action_type = 'INPUT' |
| 58 | elif ex_action_type == ActionType.STATUS_TASK_COMPLETE: |
| 59 | action = 'stop and set the query as completed' |
| 60 | action_type = 'STOP' |
| 61 | elif ex_action_type == ActionType.STATUS_TASK_IMPOSSIBLE: |
| 62 | action = 'stop and set the query as impossible' |
| 63 | action_type = 'STOP' |
| 64 | elif ex_action_type == ActionType.LONG_POINT: |
| 65 | lift_yx = json.loads(example['result_lift_yx']) |
| 66 | w, h = example['image_width'], example['image_height'] |
| 67 | assert w and h, "Invalid image size" |
| 68 | click_y, click_x = round(lift_yx[0] * h), round(lift_yx[1] * w) # Note: image size range, not 0-1000 |
| 69 | action = (click_y, click_x) |
| 70 | action_type = "LONG_POINT" |
| 71 | elif ex_action_type == ActionType.NO_ACTION: |
| 72 | duration_time = example['duration'] |
| 73 | action = f'no action for {duration_time} ms' |
| 74 | action_type = 'NO_ACTION' |
| 75 | else: |
| 76 | raise NotImplementedError |
| 77 | |
| 78 | return action, action_type |
| 79 |
nothing calls this directly
no test coverage detected