Function load

scripts/eval_step_attack.py:202–251

(example: dict, attack: str, use_caption: bool, args)

def load(example: dict, attack: str, use_caption: bool, args) -> tuple:
    intent = example["intent"]
    images = []
    if example["query_image"]:
        images.append(example["query_image"].convert("RGB"))
    obs_text = example["obs_text"] if use_caption else example["no_cap_obs_text"]

    if attack == "clip_attack":
        suffixes = {
            "gpt-4-vision-preview": "",
            "gemini-1.5-pro-preview-0409": "_gemini-1.5-pro-latest",
            "claude-3-opus-20240229": "_claude-3-opus-20240229",
            "gpt-4o-2024-05-13": "_gpt-4o-2024-05-13",
        }
        attack += suffixes[args.model]

    # Modify the caption
    if use_caption:
        caption_file = os.path.join("exp_data", "agent_adv", example["id"], f"{attack}_attack_caption.txt")
        with open(caption_file, "r") as f:
            caption = f.read().strip()
        modified_obs_text = replace_target_caption(obs_text, example["victim_som_id"], caption)
        print(f"Target caption: {example['target_caption']}")
    else:
        modified_obs_text = obs_text
    print(f"Modified observation text: {modified_obs_text}")

    # Modify the screenshot
    modified_screenshot = example["obs_screenshot"]
    if attack != "none":
        adv_image_file = os.path.join("exp_data", "agent_adv", example["id"], f"{attack}_attack_image.png")
        adv_image = Image.open(adv_image_file)
        # Resize the adversarial image and paste it on the screenshot
        x, y = example["position"]["position"]
        w, h = example["position"]["size"]
        adv_image = adv_image.resize((w, h))
        modified_screenshot.paste(adv_image, (x, y))
        # Visualize the modified screenshot
        # plt.imshow(modified_screenshot)
        # plt.show()

    trajectory = [
        {
            "observation": {"text": modified_obs_text, "image": np.array(modified_screenshot)},
            "info": {"page": DetachedPage(example["obs_url"], "")},
        }
    ]
    meta_data = {"action_history": [example["previous_action"]]}

    return intent, images, trajectory, meta_data


@beartype