| 998 | } |
| 999 | |
| 1000 | void test_batch_normalize() |
| 1001 | { |
| 1002 | using namespace dlib::tt; |
| 1003 | print_spinner(); |
| 1004 | resizable_tensor src, gamma, beta, dest, dest2, dest3, means, vars, gradient_input; |
| 1005 | src = matrix_cast<float>(gaussian_randm(5,5, 0) + 10); |
| 1006 | gamma = matrix_cast<float>(gaussian_randm(1,5, 1)); |
| 1007 | beta = matrix_cast<float>(gaussian_randm(1,5, 2)); |
| 1008 | gradient_input = matrix_cast<float>(gaussian_randm(5,5, 3)); |
| 1009 | |
| 1010 | gamma = 1; |
| 1011 | beta = 0; |
| 1012 | |
| 1013 | resizable_tensor running_means; |
| 1014 | resizable_tensor running_variances; |
| 1015 | batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1016 | const double scale = (src.num_samples())/(src.num_samples()-1.0); |
| 1017 | // Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary. |
| 1018 | running_variances = mat(running_variances)/scale; |
| 1019 | batch_normalize_inference(DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances); |
| 1020 | DLIB_TEST_MSG(max(abs(mat(dest2)-mat(dest))) < 1e-5, max(abs(mat(dest2)-mat(dest)))); |
| 1021 | cpu::batch_normalize_inference(DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances); |
| 1022 | DLIB_TEST_MSG(max(abs(mat(dest3)-mat(dest))) < 1e-5, max(abs(mat(dest3)-mat(dest)))); |
| 1023 | |
| 1024 | |
| 1025 | auto grad_src = [&](long idx) { |
| 1026 | auto f = [&](float eps) { |
| 1027 | const float old = src.host()[idx]; |
| 1028 | src.host()[idx] += eps; |
| 1029 | batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1030 | float result = dot(gradient_input, dest); |
| 1031 | src.host()[idx] = old; |
| 1032 | return result; |
| 1033 | }; |
| 1034 | const float eps = 0.01; |
| 1035 | return (f(+eps)-f(-eps))/(2*eps); |
| 1036 | }; |
| 1037 | auto grad_gamma = [&](long idx) { |
| 1038 | auto f = [&](float eps) { |
| 1039 | const float old = gamma.host()[idx]; |
| 1040 | gamma.host()[idx] += eps; |
| 1041 | batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1042 | float result = dot(gradient_input, dest); |
| 1043 | gamma.host()[idx] = old; |
| 1044 | return result; |
| 1045 | }; |
| 1046 | const float eps = 0.01; |
| 1047 | return (f(+eps)-f(-eps))/(2*eps); |
| 1048 | }; |
| 1049 | auto grad_beta = [&](long idx) { |
| 1050 | auto f = [&](float eps) { |
| 1051 | const float old = beta.host()[idx]; |
| 1052 | beta.host()[idx] += eps; |
| 1053 | batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta); |
| 1054 | float result = dot(gradient_input, dest); |
| 1055 | beta.host()[idx] = old; |
| 1056 | return result; |
| 1057 | }; |
no test coverage detected