GELU activation function.
(&self, x: &Tensor)
| 488 | |
| 489 | /// GELU activation function. |
| 490 | fn gelu(&self, x: &Tensor) -> Result<Tensor> { |
| 491 | // Fast path: raw f32 GELU approximation (tanh-based, matches PyTorch) |
| 492 | if x.dtype() == DType::F32 { |
| 493 | let data = x.contiguous()?.flatten_all()?.to_vec1::<f32>()?; |
| 494 | let shape = x.dims(); |
| 495 | let mut out = data; |
| 496 | let sqrt_2_over_pi: f32 = 0.797_884_6; // sqrt(2/pi) |
| 497 | for v in out.iter_mut() { |
| 498 | let x = *v; |
| 499 | let inner = sqrt_2_over_pi * (x + 0.044715 * x * x * x); |
| 500 | *v = 0.5 * x * (1.0 + inner.tanh()); |
| 501 | } |
| 502 | return Tensor::from_vec(out, shape, x.device()); |
| 503 | } |
| 504 | x.gelu() |
| 505 | } |
| 506 | |
| 507 | /// Sigmoid activation: `1 / (1 + exp(-x))`. |
| 508 | fn sigmoid(&self, x: &Tensor) -> Result<Tensor> { |