| 1515 | } |
| 1516 | |
| 1517 | void rms_normalize_gradient( |
| 1518 | const tensor& gradient_input, |
| 1519 | const tensor& scale, |
| 1520 | const tensor& src, |
| 1521 | const tensor& gamma, |
| 1522 | tensor& src_grad, |
| 1523 | tensor& gamma_grad, |
| 1524 | resizable_tensor& dscale |
| 1525 | ) |
| 1526 | { |
| 1527 | DLIB_CASSERT(src.num_samples() == scale.size()); |
| 1528 | DLIB_CASSERT(have_same_dimensions(gamma, gamma_grad)); |
| 1529 | DLIB_CASSERT(gamma.k() == src.k()); |
| 1530 | DLIB_CASSERT(gamma.nr() == 1); |
| 1531 | DLIB_CASSERT(gamma.nc() == 1); |
| 1532 | DLIB_CASSERT(have_same_dimensions(gradient_input, src)); |
| 1533 | DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad)); |
| 1534 | |
| 1535 | const long ns = src.num_samples(); |
| 1536 | const long ks = src.k(); |
| 1537 | const long num = src.nr() * src.nc(); |
| 1538 | |
| 1539 | gamma_grad = 0; |
| 1540 | dscale.copy_size(scale); |
| 1541 | dscale = 0; |
| 1542 | |
| 1543 | auto p_grad = gradient_input.host(); |
| 1544 | auto p_src = src.host(); |
| 1545 | const auto p_gamma = gamma.host(); |
| 1546 | const auto p_gamma_grad = gamma_grad.host(); |
| 1547 | const auto p_scale = scale.host(); |
| 1548 | auto p_dscale = dscale.host(); |
| 1549 | |
| 1550 | for (long n = 0; n < ns; ++n) |
| 1551 | { |
| 1552 | const float scale_pow = -0.5f * std::pow(p_scale[n], 3.0f); |
| 1553 | for (long k = 0; k < ks; ++k) |
| 1554 | { |
| 1555 | for (long i = 0; i < num; ++i) |
| 1556 | { |
| 1557 | const float x_hat = *p_src * p_scale[n]; |
| 1558 | p_gamma_grad[k] += (*p_grad) * x_hat; |
| 1559 | |
| 1560 | const float dx = *p_grad * p_gamma[k]; |
| 1561 | p_dscale[n] += dx * *p_src * scale_pow; |
| 1562 | |
| 1563 | ++p_grad; |
| 1564 | ++p_src; |
| 1565 | } |
| 1566 | } |
| 1567 | } |
| 1568 | |
| 1569 | p_grad = gradient_input.host(); |
| 1570 | p_src = src.host(); |
| 1571 | auto p_src_grad = src_grad.host(); |
| 1572 | const float invnum = 1.0f / (ks * num); |
| 1573 | for (long n = 0; n < ns; ++n) |
| 1574 | { |