MCPcopy Create free account
hub / github.com/davisking/dlib / rms_normalize_gradient

Function rms_normalize_gradient

dlib/cuda/cpu_dlib.cpp:1517–1588  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1515 }
1516
1517 void rms_normalize_gradient(
1518 const tensor& gradient_input,
1519 const tensor& scale,
1520 const tensor& src,
1521 const tensor& gamma,
1522 tensor& src_grad,
1523 tensor& gamma_grad,
1524 resizable_tensor& dscale
1525 )
1526 {
1527 DLIB_CASSERT(src.num_samples() == scale.size());
1528 DLIB_CASSERT(have_same_dimensions(gamma, gamma_grad));
1529 DLIB_CASSERT(gamma.k() == src.k());
1530 DLIB_CASSERT(gamma.nr() == 1);
1531 DLIB_CASSERT(gamma.nc() == 1);
1532 DLIB_CASSERT(have_same_dimensions(gradient_input, src));
1533 DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad));
1534
1535 const long ns = src.num_samples();
1536 const long ks = src.k();
1537 const long num = src.nr() * src.nc();
1538
1539 gamma_grad = 0;
1540 dscale.copy_size(scale);
1541 dscale = 0;
1542
1543 auto p_grad = gradient_input.host();
1544 auto p_src = src.host();
1545 const auto p_gamma = gamma.host();
1546 const auto p_gamma_grad = gamma_grad.host();
1547 const auto p_scale = scale.host();
1548 auto p_dscale = dscale.host();
1549
1550 for (long n = 0; n < ns; ++n)
1551 {
1552 const float scale_pow = -0.5f * std::pow(p_scale[n], 3.0f);
1553 for (long k = 0; k < ks; ++k)
1554 {
1555 for (long i = 0; i < num; ++i)
1556 {
1557 const float x_hat = *p_src * p_scale[n];
1558 p_gamma_grad[k] += (*p_grad) * x_hat;
1559
1560 const float dx = *p_grad * p_gamma[k];
1561 p_dscale[n] += dx * *p_src * scale_pow;
1562
1563 ++p_grad;
1564 ++p_src;
1565 }
1566 }
1567 }
1568
1569 p_grad = gradient_input.host();
1570 p_src = src.host();
1571 auto p_src_grad = src_grad.host();
1572 const float invnum = 1.0f / (ks * num);
1573 for (long n = 0; n < ns; ++n)
1574 {

Callers 2

test_rms_normalizeFunction · 0.50
backwardMethod · 0.50

Calls 8

copy_sizeMethod · 0.80
have_same_dimensionsFunction · 0.70
num_samplesMethod · 0.45
sizeMethod · 0.45
kMethod · 0.45
nrMethod · 0.45
ncMethod · 0.45
hostMethod · 0.45

Tested by 1

test_rms_normalizeFunction · 0.40