| 2469 | // ---------------------------------------------------------------------------------------- |
| 2470 | |
| 2471 | void compare_bn_gpu_and_cpu() |
| 2472 | { |
| 2473 | print_spinner(); |
| 2474 | resizable_tensor dest, dest2; |
| 2475 | resizable_tensor means, means2; |
| 2476 | resizable_tensor invstds, invstds2; |
| 2477 | resizable_tensor running_means, running_means2; |
| 2478 | resizable_tensor running_variances, running_variances2; |
| 2479 | resizable_tensor src(64,20,100,100); |
| 2480 | resizable_tensor gamma(1,20,100,100); |
| 2481 | resizable_tensor beta(1,20,100,100); |
| 2482 | gamma = 2; |
| 2483 | beta = 3; |
| 2484 | tt::tensor_rand rnd; |
| 2485 | rnd.fill_uniform(src); |
| 2486 | |
| 2487 | |
| 2488 | cpu::batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, invstds, 1, running_means, running_variances, src, gamma, beta); |
| 2489 | cuda::batch_normalize(DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2, 1, running_means2, running_variances2, src, gamma, beta); |
| 2490 | |
| 2491 | dlog << LINFO << "dest error: "<< max(abs(mat(dest) -mat(dest2))); |
| 2492 | dlog << LINFO << "means error: "<< max(abs(mat(means) -mat(means2))); |
| 2493 | dlog << LINFO << "invstds error: "<< max(abs(mat(invstds) -mat(invstds2))); |
| 2494 | dlog << LINFO << "running_means error: "<< max(abs(mat(running_means) -mat(running_means2))); |
| 2495 | dlog << LINFO << "running_variances error: "<< max(abs(mat(running_variances) -mat(running_variances2))); |
| 2496 | |
| 2497 | DLIB_TEST(max(abs(mat(dest) -mat(dest2))) < 1e-4); |
| 2498 | DLIB_TEST(max(abs(mat(means) -mat(means2))) < 1e-4); |
| 2499 | DLIB_TEST(max(abs(mat(invstds) -mat(invstds2))) < 1e-4); |
| 2500 | DLIB_TEST(max(abs(mat(running_means) -mat(running_means2))) < 1e-4); |
| 2501 | DLIB_TEST_MSG(max(abs(mat(running_variances) -mat(running_variances2))) < 1e-4, |
| 2502 | mean(mat(running_variances)) |
| 2503 | << "\n" << mean(mat(running_variances2)) |
| 2504 | << "\n" << max(abs(mat(running_variances) -mat(running_variances2))) |
| 2505 | << "\n" << mean(abs(mat(running_variances) -mat(running_variances2))) |
| 2506 | ); |
| 2507 | |
| 2508 | |
| 2509 | // now check that the gradients match as well |
| 2510 | resizable_tensor gradient_input; |
| 2511 | resizable_tensor src_grad, gamma_grad, beta_grad; |
| 2512 | resizable_tensor src_grad2, gamma_grad2, beta_grad2; |
| 2513 | gradient_input.copy_size(dest); |
| 2514 | src_grad.copy_size(src); src_grad = 0; src_grad2 = src_grad; |
| 2515 | gamma_grad.copy_size(gamma); gamma_grad = 0; gamma_grad2 = gamma_grad; |
| 2516 | beta_grad.copy_size(beta); beta_grad = 0; beta_grad2 = beta_grad; |
| 2517 | rnd.fill_uniform(gradient_input); |
| 2518 | |
| 2519 | |
| 2520 | cpu::batch_normalize_gradient(DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); |
| 2521 | cuda::batch_normalize_gradient(DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2); |
| 2522 | |
| 2523 | dlog << LINFO << "src_grad error: " << max(abs(mat(src_grad)-mat(src_grad2))); |
| 2524 | dlog << LINFO << "gamma_grad error: " << max(abs(mat(gamma_grad)-mat(gamma_grad2))); |
| 2525 | dlog << LINFO << "beta_grad error: " << max(abs(mat(beta_grad)-mat(beta_grad2))); |
| 2526 | DLIB_TEST(max(abs(mat(src_grad)-mat(src_grad2))) < 1e-4); |
| 2527 | DLIB_TEST(max(abs(mat(gamma_grad)-mat(gamma_grad2))) < 1e-4); |
| 2528 | DLIB_TEST(max(abs(mat(beta_grad)-mat(beta_grad2))) < 1e-4); |
no test coverage detected