MCPcopy Create free account
hub / github.com/davisking/dlib / layer_normalize_gradient

Function layer_normalize_gradient

dlib/cuda/cpu_dlib.cpp:1346–1450  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1344 }
1345
1346 void layer_normalize_gradient (
1347 const double eps,
1348 const tensor& gradient_input,
1349 const tensor& means,
1350 const tensor& invstds,
1351 const tensor& src,
1352 const tensor& gamma,
1353 tensor& src_grad,
1354 tensor& gamma_grad,
1355 tensor& beta_grad,
1356 resizable_tensor& dmeans,
1357 resizable_tensor& dvars
1358 )
1359 {
1360 const long num = src.nr() * src.nc();
1361 DLIB_CASSERT(src.num_samples() == means.size());
1362 DLIB_CASSERT(src.num_samples() == invstds.size());
1363 DLIB_CASSERT(have_same_dimensions(gamma, gamma_grad));
1364 DLIB_CASSERT(have_same_dimensions(gamma_grad, beta_grad));
1365 DLIB_CASSERT(gamma.k() == src.k());
1366 DLIB_CASSERT(gamma.nr() == 1);
1367 DLIB_CASSERT(gamma.nc() == 1);
1368 DLIB_CASSERT(have_same_dimensions(gradient_input, src));
1369 DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad));
1370 DLIB_CASSERT(eps > 0);
1371
1372 beta_grad = 0;
1373 gamma_grad = 0;
1374
1375 auto p_grad = gradient_input.host();
1376 auto p_src = src.host();
1377 const auto p_gamma = gamma.host();
1378 const auto p_gamma_grad = gamma_grad.host();
1379 const auto p_beta_grad = beta_grad.host();
1380 const auto p_invstds = invstds.host();
1381 const auto p_means = means.host();
1382
1383 dvars.copy_size(invstds);
1384 dmeans.copy_size(means);
1385 dvars = 0;
1386 dmeans = 0;
1387 const auto p_dvars = dvars.host();
1388 const auto p_dmeans = dmeans.host();
1389
1390 for (long n = 0; n < src.num_samples(); ++n)
1391 {
1392 const float invstd_pow = -0.5 * std::pow(p_invstds[n], 3.0f);
1393 for (long k = 0; k < src.k(); ++k)
1394 {
1395 for (long i = 0; i < num; ++i)
1396 {
1397 const float x_hat = (*p_src - p_means[n]) * p_invstds[n];
1398 p_beta_grad[k] += *p_grad;
1399 p_gamma_grad[k] += (*p_grad) * x_hat;
1400
1401 const float dx = *p_grad * p_gamma[k];
1402
1403 p_dvars[n] += dx * (*p_src - p_means[n]) * invstd_pow;

Callers 2

test_layer_normalizeFunction · 0.50
backwardMethod · 0.50

Calls 8

copy_sizeMethod · 0.80
have_same_dimensionsFunction · 0.70
nrMethod · 0.45
ncMethod · 0.45
num_samplesMethod · 0.45
sizeMethod · 0.45
kMethod · 0.45
hostMethod · 0.45

Tested by 1

test_layer_normalizeFunction · 0.40