Function run

scripts/run_clip_attack.py:150–220

(args: argparse.Namespace, dataset)

@beartype
def run(args: argparse.Namespace, dataset) -> None:
    attack_fn = get_attack_fn(args.attack)

    idx = 0
    for example in dataset:
        if "target_caption_clip" not in example:
            continue

        if args.index is not None and (
            idx not in list(range(args.index * args.batch_size, (args.index + 1) * args.batch_size))
        ):
            idx += 1
            continue
        idx += 1
        print(f"Running attack on example {example['id']}")

        victim_image = example["victim_image"]
        target_caption_clip = example["target_caption_clip"]
        victim_caption_clip = example["victim_caption_clip"]

        all_images = []
        all_captions = []
        for size in [180]:
            attack_out_dict = attack_fn(victim_image, target_caption_clip, victim_caption_clip, iters=1000, size=size)
            adv_images = attack_out_dict["adv_images"]

            # Evaluate with GPT-4V
            model = get_model(args.model)
            prompt_fn = model.get_captioning_prompt_fn()
            for step, adv_image in adv_images.items():
                all_images.append(adv_image)

                while True:
                    try:
                        gen_text = model.generate_answer(
                            [adv_image],
                            [prompt_fn()],
                        )[0]
                        break
                    except Exception as e:
                        print(e)
                        # Sleep a random time between 30-90 seconds
                        time.sleep(30 + 60 * random.random())
                print(f"Generated caption ({example['id']}, step {step}, size {size}): {gen_text}")
                all_captions.append(gen_text)

        # Choose the best image and caption
        best_idx = get_best_idx(all_captions, target_caption_clip, victim_caption_clip)
        adv_image = all_images[best_idx]
        adv_caption = all_captions[best_idx]

        # Save the image
        adv_image.save(
            os.path.join(
                "exp_data",
                "agent_adv",
                example["id"],
                f"{args.attack}_{f'{args.model}_' if args.model != 'gpt-4-vision-preview' else ''}attack_image.png",