| 2169 | // ---------------------------------------------------------------------------------------- |
| 2170 | |
| 2171 | void gelu ( |
| 2172 | tensor& dest, |
| 2173 | const tensor& src |
| 2174 | ) |
| 2175 | { |
| 2176 | const auto d = dest.host(); |
| 2177 | const auto s = src.host(); |
| 2178 | for (size_t i = 0; i < src.size(); ++i) |
| 2179 | d[i] = 0.5f*s[i]*(1.0f + std::erf(s[i]/sqrt_2)); |
| 2180 | } |
| 2181 | |
| 2182 | void gelu_gradient ( |
| 2183 | tensor& grad, |