| 878 | } |
| 879 | |
| 880 | void batch_normalize_gradient ( |
| 881 | const double eps, |
| 882 | const tensor& gradient_input, |
| 883 | const tensor& means, |
| 884 | const tensor& invstds, |
| 885 | const tensor& src, |
| 886 | const tensor& gamma, |
| 887 | tensor& src_grad, |
| 888 | tensor& gamma_grad, |
| 889 | tensor& beta_grad |
| 890 | ) |
| 891 | { |
| 892 | |
| 893 | const long num = src.k()*src.nr()*src.nc(); |
| 894 | DLIB_CASSERT(src.num_samples() > 1); |
| 895 | DLIB_CASSERT(num == (long)means.size()); |
| 896 | DLIB_CASSERT(num == (long)invstds.size()); |
| 897 | DLIB_CASSERT(num == (long)gamma.size()); |
| 898 | DLIB_CASSERT(num == (long)gamma_grad.size()); |
| 899 | DLIB_CASSERT(num == (long)beta_grad.size()); |
| 900 | DLIB_CASSERT(have_same_dimensions(gradient_input, src)); |
| 901 | DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad)); |
| 902 | DLIB_CASSERT(eps > 0); |
| 903 | |
| 904 | beta_grad = 0; |
| 905 | gamma_grad = 0; |
| 906 | auto p_grad = gradient_input.host(); |
| 907 | auto p_src = src.host(); |
| 908 | const auto p_gamma = gamma.host(); |
| 909 | const auto p_gamma_grad = gamma_grad.host(); |
| 910 | const auto p_beta_grad = beta_grad.host(); |
| 911 | const auto p_invstds = invstds.host(); |
| 912 | const auto p_means = means.host(); |
| 913 | |
| 914 | resizable_tensor dvars, dmeans; |
| 915 | dvars.copy_size(invstds); |
| 916 | dmeans.copy_size(means); |
| 917 | dvars = 0; |
| 918 | dmeans = 0; |
| 919 | const auto p_dvars = dvars.host(); |
| 920 | const auto p_dmeans = dmeans.host(); |
| 921 | |
| 922 | for (long n = 0; n < src.num_samples(); ++n) |
| 923 | { |
| 924 | for (long i = 0; i < num; ++i) |
| 925 | { |
| 926 | const float x_hat = (*p_src - p_means[i])*p_invstds[i]; |
| 927 | p_beta_grad[i] += *p_grad; |
| 928 | p_gamma_grad[i] += (*p_grad)*x_hat; |
| 929 | |
| 930 | const float dx = *p_grad * p_gamma[i]; |
| 931 | |
| 932 | p_dvars[i] += dx*(*p_src - p_means[i])*-0.5*std::pow(p_invstds[i], 3.0f); |
| 933 | |
| 934 | ++p_grad; |
| 935 | ++p_src; |
| 936 | } |
| 937 | } |