| 1155 | } |
| 1156 | |
| 1157 | void batch_normalize_conv_gradient( |
| 1158 | const double eps, |
| 1159 | const tensor& gradient_input, |
| 1160 | const tensor& means, |
| 1161 | const tensor& invstds, |
| 1162 | const tensor& src, |
| 1163 | const tensor& gamma, |
| 1164 | tensor& src_grad, |
| 1165 | tensor& gamma_grad, |
| 1166 | tensor& beta_grad |
| 1167 | ) |
| 1168 | { |
| 1169 | |
| 1170 | const long num = src.nr()*src.nc(); |
| 1171 | DLIB_CASSERT(src.num_samples() > 1); |
| 1172 | DLIB_CASSERT(src.k() == (long)means.size()); |
| 1173 | DLIB_CASSERT(src.k() == (long)invstds.size()); |
| 1174 | DLIB_CASSERT(src.k() == (long)gamma.size()); |
| 1175 | DLIB_CASSERT(src.k() == (long)gamma_grad.size()); |
| 1176 | DLIB_CASSERT(src.k() == (long)beta_grad.size()); |
| 1177 | DLIB_CASSERT(have_same_dimensions(gradient_input, src)); |
| 1178 | DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad)); |
| 1179 | DLIB_CASSERT(eps > 0); |
| 1180 | |
| 1181 | beta_grad = 0; |
| 1182 | gamma_grad = 0; |
| 1183 | |
| 1184 | auto p_grad = gradient_input.host(); |
| 1185 | auto p_src = src.host(); |
| 1186 | const auto p_gamma = gamma.host(); |
| 1187 | const auto p_gamma_grad = gamma_grad.host(); |
| 1188 | const auto p_beta_grad = beta_grad.host(); |
| 1189 | const auto p_invstds = invstds.host(); |
| 1190 | const auto p_means = means.host(); |
| 1191 | |
| 1192 | resizable_tensor dvars, dmeans; |
| 1193 | dvars.copy_size(invstds); |
| 1194 | dmeans.copy_size(means); |
| 1195 | dvars = 0; |
| 1196 | dmeans = 0; |
| 1197 | const auto p_dvars = dvars.host(); |
| 1198 | const auto p_dmeans = dmeans.host(); |
| 1199 | |
| 1200 | for (long n = 0; n < src.num_samples(); ++n) |
| 1201 | { |
| 1202 | for (long k = 0; k < src.k(); ++k) |
| 1203 | { |
| 1204 | const float invstd_pow = -0.5*std::pow(p_invstds[k], 3.0f); |
| 1205 | for (long i = 0; i < num; ++i) |
| 1206 | { |
| 1207 | const float x_hat = (*p_src - p_means[k])*p_invstds[k]; |
| 1208 | p_beta_grad[k] += *p_grad; |
| 1209 | p_gamma_grad[k] += (*p_grad)*x_hat; |
| 1210 | |
| 1211 | const float dx = *p_grad * p_gamma[k]; |
| 1212 | |
| 1213 | p_dvars[k] += dx*(*p_src - p_means[k])*invstd_pow; |
| 1214 | |