(episode, image_path, data_name)
| 90 | return input_string |
| 91 | |
| 92 | def load_image(episode, image_path, data_name): |
| 93 | # resize the image proportionally so that the longer side is at most 1120 |
| 94 | def __resize__(origin_img): |
| 95 | resolution = origin_img.size |
| 96 | w,h = resolution |
| 97 | max_line_res = 1120 |
| 98 | if max_line_res is not None: |
| 99 | max_line = max_line_res |
| 100 | if h > max_line: |
| 101 | w = int(w * max_line / h) |
| 102 | h = max_line |
| 103 | if w > max_line: |
| 104 | h = int(h * max_line / w) |
| 105 | w = max_line |
| 106 | img = origin_img.resize((w,h),resample=Image.Resampling.LANCZOS) |
| 107 | return img |
| 108 | |
| 109 | image = Image.open(image_path).convert("RGB") |
| 110 | image = __resize__(image) |
| 111 | |
| 112 | if data_name == 'android_control_low_test': |
| 113 | query = episode['low_instruction'] |
| 114 | else: |
| 115 | query = episode['instruction'] |
| 116 | |
| 117 | messages = [] |
| 118 | messages.append( |
| 119 | { |
| 120 | "role": "user", |
| 121 | "content": [ |
| 122 | f"<Question>{query}</Question>\n当前屏幕截图：", |
| 123 | image |
| 124 | ] |
| 125 | } |
| 126 | ) |
| 127 | return (episode,messages) |
| 128 | |
| 129 | |
| 130 | def predict(args): |
nothing calls this directly
no test coverage detected