Downsamples the provided mask tensor to match the expected dimensions for scaled dot-product attention. If the aspect ratio of the mask does not match the aspect ratio of the output image, a warning is issued. Args: mask (`torch.FloatTensor`): Th
(mask: torch.FloatTensor, batch_size: int, num_queries: int, value_embed_dim: int)
| 929 | |
| 930 | @staticmethod |
| 931 | def downsample(mask: torch.FloatTensor, batch_size: int, num_queries: int, value_embed_dim: int): |
| 932 | """ |
| 933 | Downsamples the provided mask tensor to match the expected dimensions for scaled dot-product attention. |
| 934 | If the aspect ratio of the mask does not match the aspect ratio of the output image, a warning is issued. |
| 935 | |
| 936 | Args: |
| 937 | mask (`torch.FloatTensor`): |
| 938 | The input mask tensor generated with `IPAdapterMaskProcessor.preprocess()`. |
| 939 | batch_size (`int`): |
| 940 | The batch size. |
| 941 | num_queries (`int`): |
| 942 | The number of queries. |
| 943 | value_embed_dim (`int`): |
| 944 | The dimensionality of the value embeddings. |
| 945 | |
| 946 | Returns: |
| 947 | `torch.FloatTensor`: |
| 948 | The downsampled mask tensor. |
| 949 | |
| 950 | """ |
| 951 | o_h = mask.shape[1] |
| 952 | o_w = mask.shape[2] |
| 953 | ratio = o_w / o_h |
| 954 | mask_h = int(math.sqrt(num_queries / ratio)) |
| 955 | mask_h = int(mask_h) + int((num_queries % int(mask_h)) != 0) |
| 956 | mask_w = num_queries // mask_h |
| 957 | |
| 958 | mask_downsample = F.interpolate(mask.unsqueeze(0), size=(mask_h, mask_w), mode="bicubic").squeeze(0) |
| 959 | |
| 960 | # Repeat batch_size times |
| 961 | if mask_downsample.shape[0] < batch_size: |
| 962 | mask_downsample = mask_downsample.repeat(batch_size, 1, 1) |
| 963 | |
| 964 | mask_downsample = mask_downsample.view(mask_downsample.shape[0], -1) |
| 965 | |
| 966 | downsampled_area = mask_h * mask_w |
| 967 | # If the output image and the mask do not have the same aspect ratio, tensor shapes will not match |
| 968 | # Pad tensor if downsampled_mask.shape[1] is smaller than num_queries |
| 969 | if downsampled_area < num_queries: |
| 970 | warnings.warn( |
| 971 | "The aspect ratio of the mask does not match the aspect ratio of the output image. " |
| 972 | "Please update your masks or adjust the output size for optimal performance.", |
| 973 | UserWarning, |
| 974 | ) |
| 975 | mask_downsample = F.pad(mask_downsample, (0, num_queries - mask_downsample.shape[1]), value=0.0) |
| 976 | # Discard last embeddings if downsampled_mask.shape[1] is bigger than num_queries |
| 977 | if downsampled_area > num_queries: |
| 978 | warnings.warn( |
| 979 | "The aspect ratio of the mask does not match the aspect ratio of the output image. " |
| 980 | "Please update your masks or adjust the output size for optimal performance.", |
| 981 | UserWarning, |
| 982 | ) |
| 983 | mask_downsample = mask_downsample[:, :num_queries] |
| 984 | |
| 985 | # Repeat last dimension to match SDPA output shape |
| 986 | mask_downsample = mask_downsample.view(mask_downsample.shape[0], mask_downsample.shape[1], 1).repeat( |
| 987 | 1, 1, value_embed_dim |
| 988 | ) |