| 1232 | // ---------------------------------------------------------------------------------------- |
| 1233 | |
| 1234 | void test_rms_normalize() |
| 1235 | { |
| 1236 | resizable_tensor x(2, 3, 4, 5); |
| 1237 | resizable_tensor y_cpu(x); |
| 1238 | tt::tensor_rand rnd(0); |
| 1239 | rnd.fill_uniform(x); |
| 1240 | resizable_tensor scale_cpu; |
| 1241 | resizable_tensor gamma(1, x.k()); |
| 1242 | gamma = 1; |
| 1243 | const float eps = 1e-5; |
| 1244 | cpu::rms_normalize(eps, y_cpu, scale_cpu, x, gamma); |
| 1245 | |
| 1246 | // check that the output is correctly normalized |
| 1247 | const float* p_x = x.host(); |
| 1248 | const float* p_y = y_cpu.host(); |
| 1249 | const float* p_scale = scale_cpu.host(); |
| 1250 | bool error_found = false; |
| 1251 | for (long n = 0; n < x.num_samples(); ++n) |
| 1252 | { |
| 1253 | for (long k = 0; k < x.k(); ++k) |
| 1254 | { |
| 1255 | for (long r = 0; r < x.nr(); ++r) |
| 1256 | { |
| 1257 | for (long c = 0; c < x.nc(); ++c) |
| 1258 | { |
| 1259 | float x_val = p_x[tensor_index(x, n, k, r, c)]; |
| 1260 | float y_val = p_y[tensor_index(y_cpu, n, k, r, c)]; |
| 1261 | float rms_val = p_scale[n]; |
| 1262 | if (std::abs(y_val - x_val * rms_val) >= 1e-5) error_found = true; |
| 1263 | } |
| 1264 | } |
| 1265 | } |
| 1266 | } |
| 1267 | DLIB_TEST(!error_found); |
| 1268 | |
| 1269 | // check the backward pass |
| 1270 | resizable_tensor gradient_input(x); |
| 1271 | resizable_tensor src_grad_cpu(x), gamma_grad_cpu(1, x.k()); |
| 1272 | resizable_tensor dscale_cpu(x.num_samples()); |
| 1273 | rnd.fill_gaussian(gradient_input); |
| 1274 | src_grad_cpu = 0; |
| 1275 | cpu::rms_normalize_gradient(gradient_input, scale_cpu, x, gamma, src_grad_cpu, gamma_grad_cpu, dscale_cpu); |
| 1276 | |
| 1277 | const float* p_gradient_input = gradient_input.host(); |
| 1278 | const float* p_src = x.host(); |
| 1279 | const float* p_src_grad_cpu = src_grad_cpu.host(); |
| 1280 | const float* p_gamma = gamma.host(); |
| 1281 | const float* p_scale_cpu = scale_cpu.host(); |
| 1282 | const float* p_dscale_cpu = dscale_cpu.host(); |
| 1283 | |
| 1284 | bool backward_error_found = false; |
| 1285 | for (long n = 0; n < x.num_samples(); ++n) |
| 1286 | { |
| 1287 | const float scale_pow = -0.5 * std::pow(p_scale_cpu[n], 3.0f); |
| 1288 | for (long k = 0; k < x.k(); ++k) |
| 1289 | { |
| 1290 | for (long r = 0; r < x.nr(); ++r) |
| 1291 | { |
no test coverage detected