| 89 | } |
| 90 | |
| 91 | fn stable_softplus(&self, x: &Tensor) -> Result<Tensor> { |
| 92 | // ln(1 + exp(clamp(x, -inf, 88))) with max(x, result) |
| 93 | if x.dtype() == DType::F32 { |
| 94 | let data = x.contiguous()?.flatten_all()?.to_vec1::<f32>()?; |
| 95 | let shape = x.dims(); |
| 96 | let mut out = data; |
| 97 | for v in out.iter_mut() { |
| 98 | let clamped = v.min(88.0); |
| 99 | let sp = (1.0 + clamped.exp()).ln(); |
| 100 | *v = v.max(sp); |
| 101 | } |
| 102 | return Tensor::from_vec(out, shape, x.device()); |
| 103 | } |
| 104 | let t88 = Tensor::full(88.0f32, x.shape(), x.device())?.to_dtype(x.dtype())?; |
| 105 | let clamped = x.minimum(&t88)?; |
| 106 | let sp = (clamped.exp()? + 1.0)?.log()?; |
| 107 | x.maximum(&sp) |
| 108 | } |
| 109 | |
| 110 | fn rms_norm_gated( |
| 111 | &self, |