| 1896 | // ---------------------------------------------------------------------------------------- |
| 1897 | |
| 1898 | void compare_bn_gpu_and_cpu() |
| 1899 | { |
| 1900 | print_spinner(); |
| 1901 | resizable_tensor dest, dest2; |
| 1902 | resizable_tensor means, means2; |
| 1903 | resizable_tensor invstds, invstds2; |
| 1904 | resizable_tensor running_means, running_means2; |
| 1905 | resizable_tensor running_variances, running_variances2; |
| 1906 | resizable_tensor src(64,20,100,100); |
| 1907 | resizable_tensor gamma(1,20,100,100); |
| 1908 | resizable_tensor beta(1,20,100,100); |
| 1909 | gamma = 2; |
| 1910 | beta = 3; |
| 1911 | tt::tensor_rand rnd; |
| 1912 | rnd.fill_uniform(src); |
| 1913 | |
| 1914 | |
| 1915 | cpu::batch_normalize(DEFAULT_BATCH_NORM_EPS,dest, means, invstds, 1, running_means, running_variances, src, gamma, beta); |
| 1916 | cuda::batch_normalize(DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2, 1, running_means2, running_variances2, src, gamma, beta); |
| 1917 | |
| 1918 | dlog << LINFO << "dest error: "<< max(abs(mat(dest) -mat(dest2))); |
| 1919 | dlog << LINFO << "means error: "<< max(abs(mat(means) -mat(means2))); |
| 1920 | dlog << LINFO << "invstds error: "<< max(abs(mat(invstds) -mat(invstds2))); |
| 1921 | dlog << LINFO << "running_means error: "<< max(abs(mat(running_means) -mat(running_means2))); |
| 1922 | dlog << LINFO << "running_variances error: "<< max(abs(mat(running_variances) -mat(running_variances2))); |
| 1923 | |
| 1924 | DLIB_TEST(max(abs(mat(dest) -mat(dest2))) < 1e-4); |
| 1925 | DLIB_TEST(max(abs(mat(means) -mat(means2))) < 1e-4); |
| 1926 | DLIB_TEST(max(abs(mat(invstds) -mat(invstds2))) < 1e-4); |
| 1927 | DLIB_TEST(max(abs(mat(running_means) -mat(running_means2))) < 1e-4); |
| 1928 | DLIB_TEST_MSG(max(abs(mat(running_variances) -mat(running_variances2))) < 1e-4, |
| 1929 | mean(mat(running_variances)) |
| 1930 | << "\n" << mean(mat(running_variances2)) |
| 1931 | << "\n" << max(abs(mat(running_variances) -mat(running_variances2))) |
| 1932 | << "\n" << mean(abs(mat(running_variances) -mat(running_variances2))) |
| 1933 | ); |
| 1934 | |
| 1935 | |
| 1936 | // now check that the gradients match as well |
| 1937 | resizable_tensor gradient_input; |
| 1938 | resizable_tensor src_grad, gamma_grad, beta_grad; |
| 1939 | resizable_tensor src_grad2, gamma_grad2, beta_grad2; |
| 1940 | gradient_input.copy_size(dest); |
| 1941 | src_grad.copy_size(src); src_grad = 0; src_grad2 = src_grad; |
| 1942 | gamma_grad.copy_size(gamma); gamma_grad = 0; gamma_grad2 = gamma_grad; |
| 1943 | beta_grad.copy_size(beta); beta_grad = 0; beta_grad2 = beta_grad; |
| 1944 | rnd.fill_uniform(gradient_input); |
| 1945 | |
| 1946 | |
| 1947 | cpu::batch_normalize_gradient(DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); |
| 1948 | cuda::batch_normalize_gradient(DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2); |
| 1949 | |
| 1950 | dlog << LINFO << "src_grad error: " << max(abs(mat(src_grad)-mat(src_grad2))); |
| 1951 | dlog << LINFO << "gamma_grad error: " << max(abs(mat(gamma_grad)-mat(gamma_grad2))); |
| 1952 | dlog << LINFO << "beta_grad error: " << max(abs(mat(beta_grad)-mat(beta_grad2))); |
| 1953 | DLIB_TEST(max(abs(mat(src_grad)-mat(src_grad2))) < 1e-4); |
| 1954 | DLIB_TEST(max(abs(mat(gamma_grad)-mat(gamma_grad2))) < 1e-4); |
| 1955 | DLIB_TEST(max(abs(mat(beta_grad)-mat(beta_grad2))) < 1e-4); |
no test coverage detected