MCPcopy Create free account
hub / github.com/davisking/dlib / test_rms_normalize

Function test_rms_normalize

dlib/test/dnn.cpp:1234–1323  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1232// ----------------------------------------------------------------------------------------
1233
1234 void test_rms_normalize()
1235 {
1236 resizable_tensor x(2, 3, 4, 5);
1237 resizable_tensor y_cpu(x);
1238 tt::tensor_rand rnd(0);
1239 rnd.fill_uniform(x);
1240 resizable_tensor scale_cpu;
1241 resizable_tensor gamma(1, x.k());
1242 gamma = 1;
1243 const float eps = 1e-5;
1244 cpu::rms_normalize(eps, y_cpu, scale_cpu, x, gamma);
1245
1246 // check that the output is correctly normalized
1247 const float* p_x = x.host();
1248 const float* p_y = y_cpu.host();
1249 const float* p_scale = scale_cpu.host();
1250 bool error_found = false;
1251 for (long n = 0; n < x.num_samples(); ++n)
1252 {
1253 for (long k = 0; k < x.k(); ++k)
1254 {
1255 for (long r = 0; r < x.nr(); ++r)
1256 {
1257 for (long c = 0; c < x.nc(); ++c)
1258 {
1259 float x_val = p_x[tensor_index(x, n, k, r, c)];
1260 float y_val = p_y[tensor_index(y_cpu, n, k, r, c)];
1261 float rms_val = p_scale[n];
1262 if (std::abs(y_val - x_val * rms_val) >= 1e-5) error_found = true;
1263 }
1264 }
1265 }
1266 }
1267 DLIB_TEST(!error_found);
1268
1269 // check the backward pass
1270 resizable_tensor gradient_input(x);
1271 resizable_tensor src_grad_cpu(x), gamma_grad_cpu(1, x.k());
1272 resizable_tensor dscale_cpu(x.num_samples());
1273 rnd.fill_gaussian(gradient_input);
1274 src_grad_cpu = 0;
1275 cpu::rms_normalize_gradient(gradient_input, scale_cpu, x, gamma, src_grad_cpu, gamma_grad_cpu, dscale_cpu);
1276
1277 const float* p_gradient_input = gradient_input.host();
1278 const float* p_src = x.host();
1279 const float* p_src_grad_cpu = src_grad_cpu.host();
1280 const float* p_gamma = gamma.host();
1281 const float* p_scale_cpu = scale_cpu.host();
1282 const float* p_dscale_cpu = dscale_cpu.host();
1283
1284 bool backward_error_found = false;
1285 for (long n = 0; n < x.num_samples(); ++n)
1286 {
1287 const float scale_pow = -0.5 * std::pow(p_scale_cpu[n], 3.0f);
1288 for (long k = 0; k < x.k(); ++k)
1289 {
1290 for (long r = 0; r < x.nr(); ++r)
1291 {

Callers 1

run_testsMethod · 0.85

Calls 11

tensor_indexFunction · 0.85
absFunction · 0.85
rms_normalizeFunction · 0.50
rms_normalize_gradientFunction · 0.50
maxFunction · 0.50
matFunction · 0.50
kMethod · 0.45
hostMethod · 0.45
num_samplesMethod · 0.45
nrMethod · 0.45
ncMethod · 0.45

Tested by

no test coverage detected