(
self,
dim: int,
num_attention_heads: int,
attention_head_dim: int,
dropout=0.0,
cross_attention_dim: int | None = None,
activation_fn: str = "geglu",
num_embeds_ada_norm: int | None = None,
attention_bias: bool = False,
only_cross_attention: bool = False,
double_self_attention: bool = False,
upcast_attention: bool = False,
norm_elementwise_affine: bool = True,
norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'ada_norm_continuous', 'layer_norm_i2vgen'
norm_eps: float = 1e-5,
final_dropout: bool = False,
attention_type: str = "default",
positional_embeddings: str | None = None,
num_positional_embeddings: int | None = None,
ada_norm_continous_conditioning_embedding_dim: int | None = None,
ada_norm_bias: int | None = None,
ff_inner_dim: int | None = None,
ff_bias: bool = True,
attention_out_bias: bool = True,
)
| 785 | """ |
| 786 | |
| 787 | def __init__( |
| 788 | self, |
| 789 | dim: int, |
| 790 | num_attention_heads: int, |
| 791 | attention_head_dim: int, |
| 792 | dropout=0.0, |
| 793 | cross_attention_dim: int | None = None, |
| 794 | activation_fn: str = "geglu", |
| 795 | num_embeds_ada_norm: int | None = None, |
| 796 | attention_bias: bool = False, |
| 797 | only_cross_attention: bool = False, |
| 798 | double_self_attention: bool = False, |
| 799 | upcast_attention: bool = False, |
| 800 | norm_elementwise_affine: bool = True, |
| 801 | norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'ada_norm_continuous', 'layer_norm_i2vgen' |
| 802 | norm_eps: float = 1e-5, |
| 803 | final_dropout: bool = False, |
| 804 | attention_type: str = "default", |
| 805 | positional_embeddings: str | None = None, |
| 806 | num_positional_embeddings: int | None = None, |
| 807 | ada_norm_continous_conditioning_embedding_dim: int | None = None, |
| 808 | ada_norm_bias: int | None = None, |
| 809 | ff_inner_dim: int | None = None, |
| 810 | ff_bias: bool = True, |
| 811 | attention_out_bias: bool = True, |
| 812 | ): |
| 813 | super().__init__() |
| 814 | self.dim = dim |
| 815 | self.num_attention_heads = num_attention_heads |
| 816 | self.attention_head_dim = attention_head_dim |
| 817 | self.dropout = dropout |
| 818 | self.cross_attention_dim = cross_attention_dim |
| 819 | self.activation_fn = activation_fn |
| 820 | self.attention_bias = attention_bias |
| 821 | self.double_self_attention = double_self_attention |
| 822 | self.norm_elementwise_affine = norm_elementwise_affine |
| 823 | self.positional_embeddings = positional_embeddings |
| 824 | self.num_positional_embeddings = num_positional_embeddings |
| 825 | self.only_cross_attention = only_cross_attention |
| 826 | |
| 827 | # We keep these boolean flags for backward-compatibility. |
| 828 | self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero" |
| 829 | self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" |
| 830 | self.use_ada_layer_norm_single = norm_type == "ada_norm_single" |
| 831 | self.use_layer_norm = norm_type == "layer_norm" |
| 832 | self.use_ada_layer_norm_continuous = norm_type == "ada_norm_continuous" |
| 833 | |
| 834 | if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: |
| 835 | raise ValueError( |
| 836 | f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" |
| 837 | f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." |
| 838 | ) |
| 839 | |
| 840 | self.norm_type = norm_type |
| 841 | self.num_embeds_ada_norm = num_embeds_ada_norm |
| 842 | |
| 843 | if positional_embeddings and (num_positional_embeddings is None): |
| 844 | raise ValueError( |
nothing calls this directly
no test coverage detected