| 1175 | // ---------------------------------------------------------------------------------------- |
| 1176 | |
| 1177 | void test_layer_normalize() |
| 1178 | { |
| 1179 | resizable_tensor x(2, 3, 4, 5); |
| 1180 | resizable_tensor y_cpu(x); |
| 1181 | tt::tensor_rand rnd(0); |
| 1182 | rnd.fill_uniform(x); |
| 1183 | resizable_tensor means_cpu(x.num_samples()), invstds_cpu(x.num_samples()); |
| 1184 | resizable_tensor gamma(1, x.k(), 1, 1), beta(1, x.k(), 1, 1); |
| 1185 | gamma = 1; |
| 1186 | beta = 0; |
| 1187 | const float eps = 1e-5; |
| 1188 | cpu::layer_normalize(eps, y_cpu, means_cpu, invstds_cpu, x, gamma, beta); |
| 1189 | // check that the mean and var per sample are 0 and 1 |
| 1190 | const float* p = y_cpu.host(); |
| 1191 | for (long n = 0; n < y_cpu.num_samples(); ++n) |
| 1192 | { |
| 1193 | running_stats<float> rs; |
| 1194 | for (long k = 0; k < y_cpu.k(); ++k) |
| 1195 | { |
| 1196 | for (long r = 0; r < y_cpu.nr(); ++r) |
| 1197 | { |
| 1198 | for (long c = 0; c < y_cpu.nc(); ++c) |
| 1199 | { |
| 1200 | rs.add(p[tensor_index(y_cpu, n, k, r, c)]); |
| 1201 | } |
| 1202 | } |
| 1203 | } |
| 1204 | DLIB_TEST(::std::abs(rs.mean()) < 1e-6); |
| 1205 | DLIB_TEST(::std::abs(rs.stddev() - 1.0f) < 0.01); |
| 1206 | } |
| 1207 | // check that the CPU and the CUDA implementation are equivalent |
| 1208 | IF_DLIB_USE_CUDA( |
| 1209 | resizable_tensor y_cuda(x); |
| 1210 | resizable_tensor means_cuda(x.num_samples()), invstds_cuda(x.num_samples()); |
| 1211 | cuda::layer_normalize(eps, y_cuda, means_cuda, invstds_cuda, x, gamma, beta); |
| 1212 | DLIB_TEST(max(abs(mat(y_cpu) - mat(y_cuda))) < 1e-5); |
| 1213 | DLIB_TEST(max(abs(mat(means_cpu) - mat(means_cuda))) < 1e-5); |
| 1214 | DLIB_TEST(max(abs(mat(invstds_cpu) - mat(invstds_cuda))) < 1e-5); |
| 1215 | resizable_tensor gradient_input(x); |
| 1216 | resizable_tensor src_grad_cpu(x), gamma_grad_cpu(1, x.k(), 1, 1), beta_grad_cpu(1, x.k(), 1, 1); |
| 1217 | resizable_tensor src_grad_cuda(x), gamma_grad_cuda(1, x.k(), 1, 1), beta_grad_cuda(1, x.k(), 1, 1); |
| 1218 | resizable_tensor dmeans_cpu, dvars_cpu, dmeans_cuda, dvars_cuda; |
| 1219 | rnd.fill_gaussian(gradient_input); |
| 1220 | src_grad_cpu = 0; |
| 1221 | src_grad_cuda = 0; |
| 1222 | cpu::layer_normalize_gradient(eps, gradient_input, means_cpu, invstds_cpu, x, gamma, src_grad_cpu, gamma_grad_cpu, beta_grad_cpu, dmeans_cpu, dvars_cpu); |
| 1223 | cuda::layer_normalize_gradient(eps, gradient_input, means_cuda, invstds_cuda, x, gamma, src_grad_cuda, gamma_grad_cuda, beta_grad_cuda, dmeans_cuda, dvars_cuda); |
| 1224 | DLIB_TEST(max(abs(mat(src_grad_cpu) - mat(src_grad_cuda))) < 1e-5); |
| 1225 | DLIB_TEST(max(abs(mat(gamma_grad_cpu) - mat(gamma_grad_cuda))) < 1e-5); |
| 1226 | DLIB_TEST(max(abs(mat(beta_grad_cpu) - mat(beta_grad_cuda))) < 1e-5); |
| 1227 | DLIB_TEST(max(abs(mat(dmeans_cpu) - mat(dmeans_cuda))) < 1e-4); |
| 1228 | DLIB_TEST(max(abs(mat(dvars_cpu) - mat(dvars_cuda))) < 1e-4); |
| 1229 | ) |
| 1230 | } |
| 1231 | |
| 1232 | // ---------------------------------------------------------------------------------------- |
| 1233 |
no test coverage detected