MCPcopy Create free account
hub / github.com/davisking/dlib / test_layer_normalize

Function test_layer_normalize

dlib/test/dnn.cpp:1177–1230  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1175// ----------------------------------------------------------------------------------------
1176
1177 void test_layer_normalize()
1178 {
1179 resizable_tensor x(2, 3, 4, 5);
1180 resizable_tensor y_cpu(x);
1181 tt::tensor_rand rnd(0);
1182 rnd.fill_uniform(x);
1183 resizable_tensor means_cpu(x.num_samples()), invstds_cpu(x.num_samples());
1184 resizable_tensor gamma(1, x.k(), 1, 1), beta(1, x.k(), 1, 1);
1185 gamma = 1;
1186 beta = 0;
1187 const float eps = 1e-5;
1188 cpu::layer_normalize(eps, y_cpu, means_cpu, invstds_cpu, x, gamma, beta);
1189 // check that the mean and var per sample are 0 and 1
1190 const float* p = y_cpu.host();
1191 for (long n = 0; n < y_cpu.num_samples(); ++n)
1192 {
1193 running_stats<float> rs;
1194 for (long k = 0; k < y_cpu.k(); ++k)
1195 {
1196 for (long r = 0; r < y_cpu.nr(); ++r)
1197 {
1198 for (long c = 0; c < y_cpu.nc(); ++c)
1199 {
1200 rs.add(p[tensor_index(y_cpu, n, k, r, c)]);
1201 }
1202 }
1203 }
1204 DLIB_TEST(::std::abs(rs.mean()) < 1e-6);
1205 DLIB_TEST(::std::abs(rs.stddev() - 1.0f) < 0.01);
1206 }
1207 // check that the CPU and the CUDA implementation are equivalent
1208 IF_DLIB_USE_CUDA(
1209 resizable_tensor y_cuda(x);
1210 resizable_tensor means_cuda(x.num_samples()), invstds_cuda(x.num_samples());
1211 cuda::layer_normalize(eps, y_cuda, means_cuda, invstds_cuda, x, gamma, beta);
1212 DLIB_TEST(max(abs(mat(y_cpu) - mat(y_cuda))) < 1e-5);
1213 DLIB_TEST(max(abs(mat(means_cpu) - mat(means_cuda))) < 1e-5);
1214 DLIB_TEST(max(abs(mat(invstds_cpu) - mat(invstds_cuda))) < 1e-5);
1215 resizable_tensor gradient_input(x);
1216 resizable_tensor src_grad_cpu(x), gamma_grad_cpu(1, x.k(), 1, 1), beta_grad_cpu(1, x.k(), 1, 1);
1217 resizable_tensor src_grad_cuda(x), gamma_grad_cuda(1, x.k(), 1, 1), beta_grad_cuda(1, x.k(), 1, 1);
1218 resizable_tensor dmeans_cpu, dvars_cpu, dmeans_cuda, dvars_cuda;
1219 rnd.fill_gaussian(gradient_input);
1220 src_grad_cpu = 0;
1221 src_grad_cuda = 0;
1222 cpu::layer_normalize_gradient(eps, gradient_input, means_cpu, invstds_cpu, x, gamma, src_grad_cpu, gamma_grad_cpu, beta_grad_cpu, dmeans_cpu, dvars_cpu);
1223 cuda::layer_normalize_gradient(eps, gradient_input, means_cuda, invstds_cuda, x, gamma, src_grad_cuda, gamma_grad_cuda, beta_grad_cuda, dmeans_cuda, dvars_cuda);
1224 DLIB_TEST(max(abs(mat(src_grad_cpu) - mat(src_grad_cuda))) < 1e-5);
1225 DLIB_TEST(max(abs(mat(gamma_grad_cpu) - mat(gamma_grad_cuda))) < 1e-5);
1226 DLIB_TEST(max(abs(mat(beta_grad_cpu) - mat(beta_grad_cuda))) < 1e-5);
1227 DLIB_TEST(max(abs(mat(dmeans_cpu) - mat(dmeans_cuda))) < 1e-4);
1228 DLIB_TEST(max(abs(mat(dvars_cpu) - mat(dvars_cuda))) < 1e-4);
1229 )
1230 }
1231
1232// ----------------------------------------------------------------------------------------
1233

Callers 1

run_testsMethod · 0.85

Calls 14

tensor_indexFunction · 0.85
absFunction · 0.85
layer_normalizeFunction · 0.50
maxFunction · 0.50
matFunction · 0.50
layer_normalize_gradientFunction · 0.50
num_samplesMethod · 0.45
kMethod · 0.45
hostMethod · 0.45
nrMethod · 0.45
ncMethod · 0.45
addMethod · 0.45

Tested by

no test coverage detected