| 1344 | } |
| 1345 | |
| 1346 | void layer_normalize_gradient ( |
| 1347 | const double eps, |
| 1348 | const tensor& gradient_input, |
| 1349 | const tensor& means, |
| 1350 | const tensor& invstds, |
| 1351 | const tensor& src, |
| 1352 | const tensor& gamma, |
| 1353 | tensor& src_grad, |
| 1354 | tensor& gamma_grad, |
| 1355 | tensor& beta_grad, |
| 1356 | resizable_tensor& dmeans, |
| 1357 | resizable_tensor& dvars |
| 1358 | ) |
| 1359 | { |
| 1360 | const long num = src.nr() * src.nc(); |
| 1361 | DLIB_CASSERT(src.num_samples() == means.size()); |
| 1362 | DLIB_CASSERT(src.num_samples() == invstds.size()); |
| 1363 | DLIB_CASSERT(have_same_dimensions(gamma, gamma_grad)); |
| 1364 | DLIB_CASSERT(have_same_dimensions(gamma_grad, beta_grad)); |
| 1365 | DLIB_CASSERT(gamma.k() == src.k()); |
| 1366 | DLIB_CASSERT(gamma.nr() == 1); |
| 1367 | DLIB_CASSERT(gamma.nc() == 1); |
| 1368 | DLIB_CASSERT(have_same_dimensions(gradient_input, src)); |
| 1369 | DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad)); |
| 1370 | DLIB_CASSERT(eps > 0); |
| 1371 | |
| 1372 | beta_grad = 0; |
| 1373 | gamma_grad = 0; |
| 1374 | |
| 1375 | auto p_grad = gradient_input.host(); |
| 1376 | auto p_src = src.host(); |
| 1377 | const auto p_gamma = gamma.host(); |
| 1378 | const auto p_gamma_grad = gamma_grad.host(); |
| 1379 | const auto p_beta_grad = beta_grad.host(); |
| 1380 | const auto p_invstds = invstds.host(); |
| 1381 | const auto p_means = means.host(); |
| 1382 | |
| 1383 | dvars.copy_size(invstds); |
| 1384 | dmeans.copy_size(means); |
| 1385 | dvars = 0; |
| 1386 | dmeans = 0; |
| 1387 | const auto p_dvars = dvars.host(); |
| 1388 | const auto p_dmeans = dmeans.host(); |
| 1389 | |
| 1390 | for (long n = 0; n < src.num_samples(); ++n) |
| 1391 | { |
| 1392 | const float invstd_pow = -0.5 * std::pow(p_invstds[n], 3.0f); |
| 1393 | for (long k = 0; k < src.k(); ++k) |
| 1394 | { |
| 1395 | for (long i = 0; i < num; ++i) |
| 1396 | { |
| 1397 | const float x_hat = (*p_src - p_means[n]) * p_invstds[n]; |
| 1398 | p_beta_grad[k] += *p_grad; |
| 1399 | p_gamma_grad[k] += (*p_grad) * x_hat; |
| 1400 | |
| 1401 | const float dx = *p_grad * p_gamma[k]; |
| 1402 | |
| 1403 | p_dvars[n] += dx * (*p_src - p_means[n]) * invstd_pow; |