(&self, x: &Tensor)
| 66 | } |
| 67 | |
| 68 | fn forward(&self, x: &Tensor) -> Result<Tensor> { |
| 69 | // x: [batch, dim, seq] |
| 70 | let residual = x.clone(); |
| 71 | |
| 72 | // Depthwise conv |
| 73 | let x = self.depthwise_conv1d(x)?; |
| 74 | |
| 75 | // Transpose to [batch, seq, dim] for pointwise operations |
| 76 | let x = x.transpose(1, 2)?; |
| 77 | |
| 78 | // LayerNorm |
| 79 | let x = self.layer_norm(&x)?; |
| 80 | |
| 81 | // Pointwise convolutions with GELU activation |
| 82 | let x = self.backend.linear_forward(&x, &self.pwconv1_weight, self.pwconv1_bias.as_ref())?; |
| 83 | let x = x.gelu_erf()?; |
| 84 | let x = self.backend.linear_forward(&x, &self.pwconv2_weight, self.pwconv2_bias.as_ref())?; |
| 85 | |
| 86 | // Apply gamma (channel-wise scale) |
| 87 | let x = x.broadcast_mul(&self.gamma)?; |
| 88 | |
| 89 | // Transpose back to [batch, dim, seq] |
| 90 | let x = x.transpose(1, 2)?; |
| 91 | |
| 92 | // Residual connection |
| 93 | Ok((&x + &residual)?) |
| 94 | } |
| 95 | |
| 96 | fn layer_norm(&self, x: &Tensor) -> Result<Tensor> { |
| 97 | // x: [batch, seq, dim] |
nothing calls this directly
no test coverage detected