(
self,
query_dim: int,
cross_attention_dim: Optional[int] = None,
heads: int = 8,
kv_heads: Optional[int] = None,
dim_head: int = 64,
dropout: float = 0.0,
bias: bool = False,
upcast_attention: bool = False,
upcast_softmax: bool = False,
cross_attention_norm: Optional[str] = None,
cross_attention_norm_num_groups: int = 32,
qk_norm: Optional[str] = None,
added_kv_proj_dim: Optional[int] = None,
added_proj_bias: Optional[bool] = True,
norm_num_groups: Optional[int] = None,
spatial_norm_dim: Optional[int] = None,
out_bias: bool = True,
scale_qk: bool = True,
only_cross_attention: bool = False,
eps: float = 1e-5,
rescale_output_factor: float = 1.0,
residual_connection: bool = False,
_from_deprecated_attn_block: bool = False,
processor: Optional["AttnProcessor"] = None,
out_dim: int = None,
out_context_dim: int = None,
context_pre_only=None,
pre_only=False,
elementwise_affine: bool = True,
is_causal: bool = False,
)
| 103 | """ |
| 104 | |
| 105 | def __init__( |
| 106 | self, |
| 107 | query_dim: int, |
| 108 | cross_attention_dim: Optional[int] = None, |
| 109 | heads: int = 8, |
| 110 | kv_heads: Optional[int] = None, |
| 111 | dim_head: int = 64, |
| 112 | dropout: float = 0.0, |
| 113 | bias: bool = False, |
| 114 | upcast_attention: bool = False, |
| 115 | upcast_softmax: bool = False, |
| 116 | cross_attention_norm: Optional[str] = None, |
| 117 | cross_attention_norm_num_groups: int = 32, |
| 118 | qk_norm: Optional[str] = None, |
| 119 | added_kv_proj_dim: Optional[int] = None, |
| 120 | added_proj_bias: Optional[bool] = True, |
| 121 | norm_num_groups: Optional[int] = None, |
| 122 | spatial_norm_dim: Optional[int] = None, |
| 123 | out_bias: bool = True, |
| 124 | scale_qk: bool = True, |
| 125 | only_cross_attention: bool = False, |
| 126 | eps: float = 1e-5, |
| 127 | rescale_output_factor: float = 1.0, |
| 128 | residual_connection: bool = False, |
| 129 | _from_deprecated_attn_block: bool = False, |
| 130 | processor: Optional["AttnProcessor"] = None, |
| 131 | out_dim: int = None, |
| 132 | out_context_dim: int = None, |
| 133 | context_pre_only=None, |
| 134 | pre_only=False, |
| 135 | elementwise_affine: bool = True, |
| 136 | is_causal: bool = False, |
| 137 | ): |
| 138 | super().__init__() |
| 139 | |
| 140 | # To prevent circular import. |
| 141 | from .normalization import FP32LayerNorm, LpNorm, RMSNorm |
| 142 | |
| 143 | self.inner_dim = out_dim if out_dim is not None else dim_head * heads |
| 144 | self.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads |
| 145 | self.query_dim = query_dim |
| 146 | self.use_bias = bias |
| 147 | self.is_cross_attention = cross_attention_dim is not None |
| 148 | self.cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim |
| 149 | self.upcast_attention = upcast_attention |
| 150 | self.upcast_softmax = upcast_softmax |
| 151 | self.rescale_output_factor = rescale_output_factor |
| 152 | self.residual_connection = residual_connection |
| 153 | self.dropout = dropout |
| 154 | self.fused_projections = False |
| 155 | self.out_dim = out_dim if out_dim is not None else query_dim |
| 156 | self.out_context_dim = out_context_dim if out_context_dim is not None else query_dim |
| 157 | self.context_pre_only = context_pre_only |
| 158 | self.pre_only = pre_only |
| 159 | self.is_causal = is_causal |
| 160 | |
| 161 | # we make use of this private variable to know whether this class is loaded |
| 162 | # with an deprecated state dict so that we can convert it on the fly |
no test coverage detected