| 397 | |
| 398 | fn silu_mul(&self, g: &Tensor, u: &Tensor) -> Result<Tensor> { (candle_nn::ops::silu(&g.contiguous()?)? * u.contiguous()?)?.contiguous() } |
| 399 | fn stable_softplus(&self, x: &Tensor) -> Result<Tensor> { let t=Tensor::full(88f32,x.shape(),x.device())?.to_dtype(x.dtype())?; x.maximum(&(x.minimum(&t)?.exp()?+1.0)?.log()?) } |
| 400 | fn rms_norm_gated(&self, x: &Tensor, z: &Tensor, w: &Tensor, e: f32) -> Result<Tensor> { (candle_nn::ops::rms_norm(&x.contiguous()?,w,e)?*candle_nn::ops::silu(&z.contiguous()?.to_dtype(x.dtype())?)?)?.contiguous() } |
| 401 | fn add_rms_norm(&self, a: &Tensor, b: &Tensor, w: &Tensor, e: f32) -> Result<(Tensor,Tensor)> { let r=(a+b)?; Ok((r.clone(),candle_nn::ops::rms_norm(&r.contiguous()?,w,e)?)) } |
| 402 | fn rms_norm_channel(&self, x: &Tensor, w: &Tensor, e: f32) -> Result<Tensor> { x.transpose(1,2)?.contiguous().and_then(|t|candle_nn::ops::rms_norm(&t,w,e))?.transpose(1,2)?.contiguous() } |