(react_data, actions, root, data_path, out_path, factor, rules, is_train, do_copy=False, use_qwen3=False)
| 315 | return decider_ss_entry_train, decider_ss_entry_val, grounder_ss_entry_train, grounder_ss_entry_val |
| 316 | |
| 317 | def create_grounder_entries_for_one_trace(react_data, actions, root, data_path, out_path, factor, rules, is_train, do_copy=False, use_qwen3=False): |
| 318 | grounder_entries = [] |
| 319 | |
| 320 | for i, react in enumerate(react_data, 1): |
| 321 | augment_rule = augment_data(react, rules) |
| 322 | grounder_aug_num_repeat = augment_num_repeat("grounder", augment_rule, is_train) |
| 323 | |
| 324 | img_path = os.path.join(root, f"{i}.jpg") |
| 325 | out_abspath, width, height = resize_and_copy_image("main", img_path, data_path, out_path, factor, do_copy) |
| 326 | |
| 327 | reasoning = react["reasoning"] |
| 328 | action_type = react["function"]["name"] |
| 329 | param = react["function"]["parameters"] |
| 330 | |
| 331 | if action_type == "click": |
| 332 | action = actions[i - 1] |
| 333 | if "position_x" in action and "position_y" in action: |
| 334 | coords = [int(action["position_x"]* factor), int(action["position_y"]* factor)] |
| 335 | target_element = param["target_element"] |
| 336 | if use_qwen3: |
| 337 | instruction = grounder_prompt_qwen3_coordinates.format(reasoning=reasoning, description=target_element) |
| 338 | rel_point = relative_point(coords, width, height) |
| 339 | output = format_qwen3_grounder_output(dict(point_2d=rel_point, label=target_element)) |
| 340 | else: |
| 341 | instruction = grounder_prompt.format(reasoning=reasoning, description=target_element) |
| 342 | output = json.dumps(dict(coordinates=coords)) |
| 343 | grounder_entries.extend(create_entries_for_one_step( |
| 344 | num_repeat=grounder_aug_num_repeat, |
| 345 | instruction=instruction, |
| 346 | output=output, |
| 347 | image_path=out_abspath |
| 348 | )) |
| 349 | else: |
| 350 | print(f"warning: action {i} has no position_x / y in {root}") |
| 351 | |
| 352 | if "bounds" in action and isinstance(action["bounds"], list) and len(action["bounds"]) == 4: |
| 353 | bbox = action["bounds"] |
| 354 | bbox = [int(x * factor) for x in bbox] |
| 355 | target_element = param["target_element"] |
| 356 | if use_qwen3: |
| 357 | instruction = grounder_prompt_qwen3_bbox.format(reasoning=reasoning, description=target_element) |
| 358 | rel_bbox = relative_bbox(bbox, width, height) |
| 359 | output = format_qwen3_grounder_output(dict(bbox_2d=rel_bbox, label=target_element)) |
| 360 | else: |
| 361 | instruction = grounder_prompt_bbox.format(reasoning=reasoning, description=target_element) |
| 362 | output = json.dumps(dict(bbox=bbox)) |
| 363 | grounder_entries.extend(create_entries_for_one_step( |
| 364 | num_repeat=grounder_aug_num_repeat, |
| 365 | instruction=instruction, |
| 366 | output=output, |
| 367 | image_path=out_abspath |
| 368 | )) |
| 369 | else: |
| 370 | print(f"warning: action {i} has no valid bounds in {root}") |
| 371 | return grounder_entries |
| 372 | |
| 373 | def create_decider_entries_for_one_task(task, react_data, actions, root, data_path, out_path, factor, rules, unexpected_img_safe_abspaths, is_train, do_copy=False, e2e=False, use_qwen3=False): |
| 374 | # decider |
no test coverage detected