| 170 | return causal_mask_bool |
| 171 | |
| 172 | class FeedForward(nn.Module): |
| 173 | def __init__( |
| 174 | self, |
| 175 | dim: int, |
| 176 | hidden_dim: int, |
| 177 | out_dim: int, |
| 178 | bias: bool, |
| 179 | multiple_of: int, |
| 180 | ): |
| 181 | super().__init__() |
| 182 | |
| 183 | hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) |
| 184 | |
| 185 | self.w1 = ColumnParallelLinear( |
| 186 | dim, hidden_dim, bias=bias, gather_output=False |
| 187 | ) |
| 188 | self.w2 = RowParallelLinear( |
| 189 | dim, hidden_dim, bias=bias, input_is_parallel=True |
| 190 | ) |
| 191 | self.w3 = ColumnParallelLinear( |
| 192 | hidden_dim, out_dim, bias=bias, gather_output=False |
| 193 | ) |
| 194 | |
| 195 | # @torch.compile |
| 196 | def _silu_gating(self, x, y): |
| 197 | return F.silu(x) * y |
| 198 | |
| 199 | def forward(self, x): |
| 200 | return self.w3(self._silu_gating(self.w1(x), self.w2(x))) |
| 201 | |
| 202 | |
| 203 | class PackedFlashBaseLayer1D(nn.Module): |