(
self,
dim: int,
num_attention_heads: int,
attention_head_dim: int,
dropout: float = 0.0,
cross_attention_dim: int | None = None,
activation_fn: str = "geglu",
num_embeds_ada_norm: int | None = None,
attention_bias: bool = False,
only_cross_attention: bool = False,
double_self_attention: bool = False,
upcast_attention: bool = False,
norm_elementwise_affine: bool = True,
norm_type: str = "layer_norm",
norm_eps: float = 1e-5,
final_dropout: bool = False,
positional_embeddings: str | None = None,
num_positional_embeddings: int | None = None,
ff_inner_dim: int | None = None,
ff_bias: bool = True,
attention_out_bias: bool = True,
context_length: int = 16,
context_stride: int = 4,
weighting_scheme: str = "pyramid",
)
| 1388 | """ |
| 1389 | |
| 1390 | def __init__( |
| 1391 | self, |
| 1392 | dim: int, |
| 1393 | num_attention_heads: int, |
| 1394 | attention_head_dim: int, |
| 1395 | dropout: float = 0.0, |
| 1396 | cross_attention_dim: int | None = None, |
| 1397 | activation_fn: str = "geglu", |
| 1398 | num_embeds_ada_norm: int | None = None, |
| 1399 | attention_bias: bool = False, |
| 1400 | only_cross_attention: bool = False, |
| 1401 | double_self_attention: bool = False, |
| 1402 | upcast_attention: bool = False, |
| 1403 | norm_elementwise_affine: bool = True, |
| 1404 | norm_type: str = "layer_norm", |
| 1405 | norm_eps: float = 1e-5, |
| 1406 | final_dropout: bool = False, |
| 1407 | positional_embeddings: str | None = None, |
| 1408 | num_positional_embeddings: int | None = None, |
| 1409 | ff_inner_dim: int | None = None, |
| 1410 | ff_bias: bool = True, |
| 1411 | attention_out_bias: bool = True, |
| 1412 | context_length: int = 16, |
| 1413 | context_stride: int = 4, |
| 1414 | weighting_scheme: str = "pyramid", |
| 1415 | ): |
| 1416 | super().__init__() |
| 1417 | self.dim = dim |
| 1418 | self.num_attention_heads = num_attention_heads |
| 1419 | self.attention_head_dim = attention_head_dim |
| 1420 | self.dropout = dropout |
| 1421 | self.cross_attention_dim = cross_attention_dim |
| 1422 | self.activation_fn = activation_fn |
| 1423 | self.attention_bias = attention_bias |
| 1424 | self.double_self_attention = double_self_attention |
| 1425 | self.norm_elementwise_affine = norm_elementwise_affine |
| 1426 | self.positional_embeddings = positional_embeddings |
| 1427 | self.num_positional_embeddings = num_positional_embeddings |
| 1428 | self.only_cross_attention = only_cross_attention |
| 1429 | |
| 1430 | self.set_free_noise_properties(context_length, context_stride, weighting_scheme) |
| 1431 | |
| 1432 | # We keep these boolean flags for backward-compatibility. |
| 1433 | self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero" |
| 1434 | self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" |
| 1435 | self.use_ada_layer_norm_single = norm_type == "ada_norm_single" |
| 1436 | self.use_layer_norm = norm_type == "layer_norm" |
| 1437 | self.use_ada_layer_norm_continuous = norm_type == "ada_norm_continuous" |
| 1438 | |
| 1439 | if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: |
| 1440 | raise ValueError( |
| 1441 | f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" |
| 1442 | f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." |
| 1443 | ) |
| 1444 | |
| 1445 | self.norm_type = norm_type |
| 1446 | self.num_embeds_ada_norm = num_embeds_ada_norm |
| 1447 |
nothing calls this directly
no test coverage detected