(
self,
query_dim: int,
cross_attention_dim: Optional[int] = None,
heads: int = 8,
dim_head: int = 64,
dropout: float = 0.0,
bias: bool = False,
upcast_attention: bool = False,
upcast_softmax: bool = False,
cross_attention_norm: Optional[str] = None,
cross_attention_norm_num_groups: int = 32,
added_kv_proj_dim: Optional[int] = None,
norm_num_groups: Optional[int] = None,
spatial_norm_dim: Optional[int] = None,
out_bias: bool = True,
scale_qk: bool = True,
only_cross_attention: bool = False,
eps: float = 1e-5,
rescale_output_factor: float = 1.0,
residual_connection: bool = False,
_from_deprecated_attn_block: bool = False,
processor: Optional["AttnProcessor"] = None,
)
| 87 | """ |
| 88 | |
| 89 | def __init__( |
| 90 | self, |
| 91 | query_dim: int, |
| 92 | cross_attention_dim: Optional[int] = None, |
| 93 | heads: int = 8, |
| 94 | dim_head: int = 64, |
| 95 | dropout: float = 0.0, |
| 96 | bias: bool = False, |
| 97 | upcast_attention: bool = False, |
| 98 | upcast_softmax: bool = False, |
| 99 | cross_attention_norm: Optional[str] = None, |
| 100 | cross_attention_norm_num_groups: int = 32, |
| 101 | added_kv_proj_dim: Optional[int] = None, |
| 102 | norm_num_groups: Optional[int] = None, |
| 103 | spatial_norm_dim: Optional[int] = None, |
| 104 | out_bias: bool = True, |
| 105 | scale_qk: bool = True, |
| 106 | only_cross_attention: bool = False, |
| 107 | eps: float = 1e-5, |
| 108 | rescale_output_factor: float = 1.0, |
| 109 | residual_connection: bool = False, |
| 110 | _from_deprecated_attn_block: bool = False, |
| 111 | processor: Optional["AttnProcessor"] = None, |
| 112 | ): |
| 113 | super().__init__() |
| 114 | self.inner_dim = dim_head * heads |
| 115 | self.cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim |
| 116 | self.upcast_attention = upcast_attention |
| 117 | self.upcast_softmax = upcast_softmax |
| 118 | self.rescale_output_factor = rescale_output_factor |
| 119 | self.residual_connection = residual_connection |
| 120 | self.dropout = dropout |
| 121 | |
| 122 | # we make use of this private variable to know whether this class is loaded |
| 123 | # with an deprecated state dict so that we can convert it on the fly |
| 124 | self._from_deprecated_attn_block = _from_deprecated_attn_block |
| 125 | |
| 126 | self.scale_qk = scale_qk |
| 127 | self.scale = dim_head**-0.5 if self.scale_qk else 1.0 |
| 128 | |
| 129 | self.heads = heads |
| 130 | # for slice_size > 0 the attention score computation |
| 131 | # is split across the batch axis to save memory |
| 132 | # You can set slice_size with `set_attention_slice` |
| 133 | self.sliceable_head_dim = heads |
| 134 | |
| 135 | self.added_kv_proj_dim = added_kv_proj_dim |
| 136 | self.only_cross_attention = only_cross_attention |
| 137 | |
| 138 | if self.added_kv_proj_dim is None and self.only_cross_attention: |
| 139 | raise ValueError( |
| 140 | "`only_cross_attention` can only be set to True if `added_kv_proj_dim` is not None. Make sure to set either `only_cross_attention=False` or define `added_kv_proj_dim`." |
| 141 | ) |
| 142 | |
| 143 | if norm_num_groups is not None: |
| 144 | self.group_norm = nn.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True) |
| 145 | else: |
| 146 | self.group_norm = None |
no test coverage detected