(model, image, inputs, outputs, epsilon=16 / 255, alpha=1 / 255, iters=4000, size=1536)
| 16 | |
| 17 | |
| 18 | def bim(model, image, inputs, outputs, epsilon=16 / 255, alpha=1 / 255, iters=4000, size=1536): |
| 19 | device = model.distributed_state.device |
| 20 | |
| 21 | # Freeze the model |
| 22 | model.freeze() |
| 23 | |
| 24 | if size: |
| 25 | image = resize_image(image, size) |
| 26 | image = torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0).permute(0, 3, 1, 2).to(device) |
| 27 | delta = torch.zeros_like(image, requires_grad=True) |
| 28 | |
| 29 | evaluate_from_tensor(model, image + delta, inputs, outputs) |
| 30 | |
| 31 | losses = [] |
| 32 | best_delta = None |
| 33 | best_acc = 0 |
| 34 | for idx in tqdm(range(iters)): |
| 35 | pixel_values = model.image_processor_from_tensor(image + delta) |
| 36 | |
| 37 | loss = model.forward(pixel_values, questions=inputs, answers=outputs, image_sizes=None) |
| 38 | # print(loss.item()) |
| 39 | losses.append(loss.item()) |
| 40 | loss.backward() |
| 41 | |
| 42 | # loss the lower the better |
| 43 | with torch.no_grad(): |
| 44 | delta.grad.sign_() |
| 45 | delta.data = delta.data - alpha * delta.grad |
| 46 | delta.data.clamp_(-epsilon, epsilon) |
| 47 | delta.data = torch.clamp(image + delta, 0, 1) - image |
| 48 | delta.grad.zero_() |
| 49 | |
| 50 | if (idx + 1) % 200 == 0: |
| 51 | with torch.no_grad(): |
| 52 | pixel_values = model.image_processor_from_tensor(image + delta) |
| 53 | loss = model.forward(pixel_values, questions=inputs, answers=outputs, image_sizes=None) |
| 54 | print("Loss:", loss.item()) |
| 55 | acc = evaluate_from_tensor(model, image + delta, inputs, outputs) |
| 56 | # Save the image |
| 57 | # image_np = (image + delta).squeeze(0).detach().cpu().numpy() |
| 58 | # image_np = (image_np * 255).astype("uint8").transpose(1, 2, 0) |
| 59 | # Image.fromarray(image_np).save(f"attack/attacks/bim_image_{idx + 1}.png") |
| 60 | # Plot the loss |
| 61 | # sns.lineplot(x=range(len(losses)), y=losses) |
| 62 | # plt.savefig(f"attack/attacks/bim_loss.png") |
| 63 | # plt.close() |
| 64 | |
| 65 | if acc > best_acc: |
| 66 | best_acc = acc |
| 67 | best_delta = delta.clone() |
| 68 | |
| 69 | # Early stopping |
| 70 | if acc == 1: |
| 71 | break |
| 72 | |
| 73 | if best_acc != 1: |
| 74 | delta = best_delta |
| 75 | image_np = (image + delta).squeeze(0).detach().cpu().numpy() |
no test coverage detected