| 2180 | } |
| 2181 | |
| 2182 | void gelu_gradient ( |
| 2183 | tensor& grad, |
| 2184 | const tensor& src, |
| 2185 | const tensor& gradient_input |
| 2186 | ) |
| 2187 | { |
| 2188 | const float beta = 1.0f / std::sqrt(2.0f * pi); |
| 2189 | const auto compute_gradient = [beta](float x) |
| 2190 | { |
| 2191 | const float cdf = 0.5f*(1.0f + std::erf(x/sqrt_2)); |
| 2192 | const float pdf = beta*std::exp(-0.5f*x*x); |
| 2193 | return cdf + x * pdf; |
| 2194 | }; |
| 2195 | const auto g = grad.host(); |
| 2196 | const auto s = src.host(); |
| 2197 | const auto in = gradient_input.host(); |
| 2198 | if (is_same_object(grad, gradient_input)) |
| 2199 | { |
| 2200 | for (size_t i = 0; i < src.size(); ++i) |
| 2201 | g[i] = in[i]*compute_gradient(s[i]); |
| 2202 | } |
| 2203 | else |
| 2204 | { |
| 2205 | for (size_t i = 0; i < src.size(); ++i) |
| 2206 | g[i] += in[i]*compute_gradient(s[i]); |
| 2207 | } |
| 2208 | } |
| 2209 | |
| 2210 | // ---------------------------------------------------------------------------------------- |
| 2211 | |