(&self, x: &Tensor)
| 46 | } |
| 47 | |
| 48 | pub fn forward(&self, x: &Tensor) -> Result<Tensor> { |
| 49 | // x: [batch, seq, dim] |
| 50 | let x = self.backend.linear_forward(x, &self.in_proj_weight, self.in_proj_bias.as_ref())?; |
| 51 | |
| 52 | // GLU gate: split into two halves along last dim |
| 53 | let half = self.dim; |
| 54 | let a = x.narrow(candle_core::D::Minus1, 0, half)?; |
| 55 | let b = x.narrow(candle_core::D::Minus1, half, half)?; |
| 56 | let gate = self.backend.sigmoid(&b)?; |
| 57 | let x = (a * gate)?; |
| 58 | |
| 59 | // Depthwise conv1d: transpose to [batch, dim, seq], apply, transpose back |
| 60 | let x = x.transpose(1, 2)?; // [batch, dim, seq] |
| 61 | let x = self.depthwise_conv1d(&x)?; |
| 62 | let x = x.transpose(1, 2)?; // [batch, seq, dim] |
| 63 | |
| 64 | // SwooshR activation before out_proj (out_proj is ActivationDropoutAndLinear with SwooshR) |
| 65 | let x = super::activations::swoosh_r(&x)?; |
| 66 | let x = self.backend.linear_forward(&x, &self.out_proj_weight, self.out_proj_bias.as_ref())?; |
| 67 | Ok(x) |
| 68 | } |
| 69 | |
| 70 | /// Manual depthwise conv1d using broadcast_mul + sum pattern. |
| 71 | fn depthwise_conv1d(&self, x: &Tensor) -> Result<Tensor> { |
nothing calls this directly
no test coverage detected