(traj_root)
| 21 | |
| 22 | |
| 23 | def get_examples(traj_root): |
| 24 | examples = [] |
| 25 | # Enumerate all directory names in the root directory |
| 26 | for dir_name in sorted(os.listdir(traj_root)): |
| 27 | if dir_name == ".DS_Store": |
| 28 | continue |
| 29 | |
| 30 | # Enumerate all files in the directory |
| 31 | example_dir = os.path.join(traj_root, dir_name) |
| 32 | base_example_dir = get_base_example_dir(example_dir) |
| 33 | |
| 34 | with open(os.path.join(example_dir, "data.json")) as f: |
| 35 | data = json.load(f) |
| 36 | |
| 37 | with open(os.path.join(base_example_dir, "obs_text.txt")) as f: |
| 38 | obs_text = f.read().strip() |
| 39 | no_cap_obs_text = remove_caption(obs_text) |
| 40 | |
| 41 | obs_screenshot = Image.open(os.path.join(base_example_dir, "obs_screenshot.png")) |
| 42 | |
| 43 | query_image = None |
| 44 | if os.path.exists(query_image_path := os.path.join(example_dir, "query_image.png")): |
| 45 | query_image = Image.open(query_image_path).convert("RGB") |
| 46 | |
| 47 | victim_image = Image.open(os.path.join(example_dir, "victim_image.png")).convert("RGB") |
| 48 | |
| 49 | example = { |
| 50 | "id": dir_name, |
| 51 | "intent": data["intent"], |
| 52 | "query_image": query_image, |
| 53 | "obs_text": obs_text, |
| 54 | "no_cap_obs_text": no_cap_obs_text, |
| 55 | "obs_screenshot": obs_screenshot, |
| 56 | "obs_url": data["obs_url"], |
| 57 | "previous_action": data["previous_action"], |
| 58 | "victim_image": victim_image, |
| 59 | "target_caption": data["target_caption"], # This is for the caption attack |
| 60 | "victim_som_id": data["victim_som_id"], |
| 61 | "target_label": data["target_label"], |
| 62 | "position": data["position"], |
| 63 | } |
| 64 | if os.path.exists(target_image_path := os.path.join(example_dir, "target_image.png")): |
| 65 | target_image = Image.open(target_image_path).convert("RGB") |
| 66 | example["target_image"] = target_image # This is for the encoder attack |
| 67 | if "target_caption_clip" in data and data["target_caption_clip"]: |
| 68 | example["target_caption_clip"] = data["target_caption_clip"] # This is for the CLIP attack |
| 69 | if "victim_caption_clip" in data: |
| 70 | example["victim_caption_clip"] = data["victim_caption_clip"] # This is for the CLIP attack |
| 71 | examples.append(example) |
| 72 | |
| 73 | return examples |
| 74 | |
| 75 | |
| 76 | if __name__ == "__main__": |
no test coverage detected