r""" A feed-forward layer. Parameters: dim (`int`): The number of channels in the input. dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden
| 1142 | |
| 1143 | |
| 1144 | class FeedForward(nn.Module): |
| 1145 | r""" |
| 1146 | A feed-forward layer. |
| 1147 | |
| 1148 | Parameters: |
| 1149 | dim (`int`): The number of channels in the input. |
| 1150 | dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. |
| 1151 | mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. |
| 1152 | dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. |
| 1153 | activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. |
| 1154 | final_dropout (`bool` *optional*, defaults to False): Apply a final dropout. |
| 1155 | bias (`bool`, defaults to True): Whether to use a bias in the linear layer. |
| 1156 | """ |
| 1157 | |
| 1158 | def __init__( |
| 1159 | self, |
| 1160 | dim: int, |
| 1161 | dim_out: Optional[int] = None, |
| 1162 | mult: int = 4, |
| 1163 | dropout: float = 0.0, |
| 1164 | activation_fn: str = "geglu", |
| 1165 | final_dropout: bool = False, |
| 1166 | inner_dim=None, |
| 1167 | bias: bool = True, |
| 1168 | ): |
| 1169 | super().__init__() |
| 1170 | if inner_dim is None: |
| 1171 | inner_dim = int(dim * mult) |
| 1172 | dim_out = dim_out if dim_out is not None else dim |
| 1173 | |
| 1174 | if activation_fn == "gelu": |
| 1175 | act_fn = GELU(dim, inner_dim, bias=bias) |
| 1176 | if activation_fn == "gelu-approximate": |
| 1177 | act_fn = GELU(dim, inner_dim, approximate="tanh", bias=bias) |
| 1178 | elif activation_fn == "geglu": |
| 1179 | act_fn = GEGLU(dim, inner_dim, bias=bias) |
| 1180 | elif activation_fn == "geglu-approximate": |
| 1181 | act_fn = ApproximateGELU(dim, inner_dim, bias=bias) |
| 1182 | elif activation_fn == "swiglu": |
| 1183 | act_fn = SwiGLU(dim, inner_dim, bias=bias) |
| 1184 | |
| 1185 | self.net = nn.ModuleList([]) |
| 1186 | # project in |
| 1187 | self.net.append(act_fn) |
| 1188 | # project dropout |
| 1189 | self.net.append(nn.Dropout(dropout)) |
| 1190 | # project out |
| 1191 | self.net.append(nn.Linear(inner_dim, dim_out, bias=bias)) |
| 1192 | # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout |
| 1193 | if final_dropout: |
| 1194 | self.net.append(nn.Dropout(dropout)) |
| 1195 | |
| 1196 | def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor: |
| 1197 | if len(args) > 0 or kwargs.get("scale", None) is not None: |
| 1198 | deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." |
| 1199 | deprecate("scale", "1.0.0", deprecation_message) |
| 1200 | for module in self.net: |
| 1201 | hidden_states = module(hidden_states) |
no outgoing calls
no test coverage detected