MCPcopy Create free account
hub / github.com/davisking/dlib / test_batch_normalize

Function test_batch_normalize

dlib/test/dnn.cpp:1000–1083  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

998 }
999
1000 void test_batch_normalize()
1001 {
1002 using namespace dlib::tt;
1003 print_spinner();
1004 resizable_tensor src, gamma, beta, dest, dest2, dest3, means, vars, gradient_input;
1005 src = matrix_cast<float>(gaussian_randm(5,5, 0) + 10);
1006 gamma = matrix_cast<float>(gaussian_randm(1,5, 1));
1007 beta = matrix_cast<float>(gaussian_randm(1,5, 2));
1008 gradient_input = matrix_cast<float>(gaussian_randm(5,5, 3));
1009
1010 gamma = 1;
1011 beta = 0;
1012
1013 resizable_tensor running_means;
1014 resizable_tensor running_variances;
1015 batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta);
1016 const double scale = (src.num_samples())/(src.num_samples()-1.0);
1017 // Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary.
1018 running_variances = mat(running_variances)/scale;
1019 batch_normalize_inference(DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances);
1020 DLIB_TEST_MSG(max(abs(mat(dest2)-mat(dest))) < 1e-5, max(abs(mat(dest2)-mat(dest))));
1021 cpu::batch_normalize_inference(DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances);
1022 DLIB_TEST_MSG(max(abs(mat(dest3)-mat(dest))) < 1e-5, max(abs(mat(dest3)-mat(dest))));
1023
1024
1025 auto grad_src = [&](long idx) {
1026 auto f = [&](float eps) {
1027 const float old = src.host()[idx];
1028 src.host()[idx] += eps;
1029 batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta);
1030 float result = dot(gradient_input, dest);
1031 src.host()[idx] = old;
1032 return result;
1033 };
1034 const float eps = 0.01;
1035 return (f(+eps)-f(-eps))/(2*eps);
1036 };
1037 auto grad_gamma = [&](long idx) {
1038 auto f = [&](float eps) {
1039 const float old = gamma.host()[idx];
1040 gamma.host()[idx] += eps;
1041 batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta);
1042 float result = dot(gradient_input, dest);
1043 gamma.host()[idx] = old;
1044 return result;
1045 };
1046 const float eps = 0.01;
1047 return (f(+eps)-f(-eps))/(2*eps);
1048 };
1049 auto grad_beta = [&](long idx) {
1050 auto f = [&](float eps) {
1051 const float old = beta.host()[idx];
1052 beta.host()[idx] += eps;
1053 batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, vars, 1, running_means, running_variances, src, gamma, beta);
1054 float result = dot(gradient_input, dest);
1055 beta.host()[idx] = old;
1056 return result;
1057 };

Callers 1

run_testsMethod · 0.85

Calls 14

print_spinnerFunction · 0.85
gaussian_randmFunction · 0.85
absFunction · 0.85
compare_gradientsFunction · 0.85
copy_sizeMethod · 0.80
fFunction · 0.70
batch_normalizeFunction · 0.50
matFunction · 0.50
maxFunction · 0.50
dotFunction · 0.50
batch_normalize_gradientFunction · 0.50

Tested by

no test coverage detected