| 1083 | } |
| 1084 | |
| 1085 | void test_batch_normalize_conv() |
| 1086 | { |
| 1087 | using namespace dlib::tt; |
| 1088 | print_spinner(); |
| 1089 | resizable_tensor src(5,5,4,4), gamma, beta, dest, dest2, dest3, means, vars, gradient_input(5,5,4,4); |
| 1090 | tt::tensor_rand rnd; |
| 1091 | rnd.fill_gaussian(src,10); |
| 1092 | rnd.fill_gaussian(gradient_input); |
| 1093 | gamma = matrix_cast<float>(gaussian_randm(1,5, 1)); |
| 1094 | beta = matrix_cast<float>(gaussian_randm(1,5, 2)); |
| 1095 | |
| 1096 | gamma = 1; |
| 1097 | beta = 0; |
| 1098 | |
| 1099 | resizable_tensor running_means; |
| 1100 | resizable_tensor running_variances; |
| 1101 | batch_normalize_conv(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1102 | const double scale = (src.num_samples()*src.nr()*src.nc())/(src.num_samples()*src.nr()*src.nc()-1.0); |
| 1103 | // Turn back into biased variance estimate because that's how |
| 1104 | // batch_normalize_conv() works, so if we want to match it this is necessary. |
| 1105 | running_variances = mat(running_variances)/scale; |
| 1106 | batch_normalize_conv_inference(DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances); |
| 1107 | DLIB_TEST(max(abs(mat(dest2)-mat(dest))) < 1e-5); |
| 1108 | cpu::batch_normalize_conv_inference(DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances); |
| 1109 | DLIB_TEST(max(abs(mat(dest3)-mat(dest))) < 1e-5); |
| 1110 | |
| 1111 | |
| 1112 | auto grad_src = [&](long idx) { |
| 1113 | auto f = [&](float eps) { |
| 1114 | const float old = src.host()[idx]; |
| 1115 | src.host()[idx] += eps; |
| 1116 | batch_normalize_conv(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1117 | float result = dot(gradient_input, dest); |
| 1118 | src.host()[idx] = old; |
| 1119 | return result; |
| 1120 | }; |
| 1121 | const float eps = 0.01; |
| 1122 | return (f(+eps)-f(-eps))/(2*eps); |
| 1123 | }; |
| 1124 | auto grad_gamma = [&](long idx) { |
| 1125 | auto f = [&](float eps) { |
| 1126 | const float old = gamma.host()[idx]; |
| 1127 | gamma.host()[idx] += eps; |
| 1128 | batch_normalize_conv(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1129 | float result = dot(gradient_input, dest); |
| 1130 | gamma.host()[idx] = old; |
| 1131 | return result; |
| 1132 | }; |
| 1133 | const float eps = 0.01; |
| 1134 | return (f(+eps)-f(-eps))/(2*eps); |
| 1135 | }; |
| 1136 | auto grad_beta = [&](long idx) { |
| 1137 | auto f = [&](float eps) { |
| 1138 | const float old = beta.host()[idx]; |
| 1139 | beta.host()[idx] += eps; |
| 1140 | batch_normalize_conv(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1141 | float result = dot(gradient_input, dest); |
| 1142 | beta.host()[idx] = old; |
no test coverage detected