MCPcopy Create free account
hub / github.com/davisking/dlib / gelu_gradient

Function gelu_gradient

dlib/cuda/cpu_dlib.cpp:2182–2208  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

2180 }
2181
2182 void gelu_gradient (
2183 tensor& grad,
2184 const tensor& src,
2185 const tensor& gradient_input
2186 )
2187 {
2188 const float beta = 1.0f / std::sqrt(2.0f * pi);
2189 const auto compute_gradient = [beta](float x)
2190 {
2191 const float cdf = 0.5f*(1.0f + std::erf(x/sqrt_2));
2192 const float pdf = beta*std::exp(-0.5f*x*x);
2193 return cdf + x * pdf;
2194 };
2195 const auto g = grad.host();
2196 const auto s = src.host();
2197 const auto in = gradient_input.host();
2198 if (is_same_object(grad, gradient_input))
2199 {
2200 for (size_t i = 0; i < src.size(); ++i)
2201 g[i] = in[i]*compute_gradient(s[i]);
2202 }
2203 else
2204 {
2205 for (size_t i = 0; i < src.size(); ++i)
2206 g[i] += in[i]*compute_gradient(s[i]);
2207 }
2208 }
2209
2210 // ----------------------------------------------------------------------------------------
2211

Callers 1

backwardMethod · 0.50

Calls 5

expFunction · 0.85
sqrtFunction · 0.50
is_same_objectFunction · 0.50
hostMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected