| 2866 | |
| 2867 | |
| 2868 | void CopyMatrixInVTransScaleRows (size_t h, size_t w, |
| 2869 | double * ps, size_t dists, |
| 2870 | double * pd, size_t distd, |
| 2871 | double * pscale, size_t distscale) |
| 2872 | { |
| 2873 | __m256i mask = my_mm256_cmpgt_epi64(_mm256_set1_epi64x(w&3), |
| 2874 | _mm256_set_epi64x(3,2,1,0)); |
| 2875 | |
| 2876 | size_t i = 0; |
| 2877 | |
| 2878 | for ( ; i < h; i+=2, pd += 8, pscale += 2*distscale) |
| 2879 | { |
| 2880 | double * psnext = ps+2*dists; |
| 2881 | __m256d scale0 = _mm256_set1_pd(*pscale); |
| 2882 | __m256d scale1 = _mm256_set1_pd(*(pscale+distscale)); |
| 2883 | |
| 2884 | size_t j = 0; |
| 2885 | double * pd2 = pd; |
| 2886 | |
| 2887 | for ( ; j+4 <= w; j+=4, pd2 += 4*distd) |
| 2888 | { |
| 2889 | _mm256_storeu_pd(pd2, scale0 * _mm256_loadu_pd(ps+j)); |
| 2890 | _mm256_storeu_pd(pd2+4, scale1 * _mm256_loadu_pd(ps+dists+j)); |
| 2891 | } |
| 2892 | _mm256_maskstore_pd (pd2, mask, scale0 * _mm256_maskload_pd(ps+j, mask)); |
| 2893 | _mm256_maskstore_pd (pd2+4, mask, scale1 * _mm256_maskload_pd(ps+dists+j, mask)); |
| 2894 | ps = psnext; |
| 2895 | } |
| 2896 | |
| 2897 | for ( ; i < h; i++, pd += 4, pscale += distscale) |
| 2898 | { |
| 2899 | double * psnext = ps+dists; |
| 2900 | __m256d scale = _mm256_set1_pd(*pscale); |
| 2901 | |
| 2902 | size_t j = 0; |
| 2903 | double * pd2 = pd; |
| 2904 | for ( ; j + 16 <= w; j+=16, pd2 += 16*distd) |
| 2905 | { |
| 2906 | Prefetch<0> (psnext+j); |
| 2907 | Prefetch<0> (psnext+j+8); |
| 2908 | auto val1 = scale * _mm256_loadu_pd(ps+j); |
| 2909 | auto val2 = scale * _mm256_loadu_pd(ps+j+4); |
| 2910 | auto val3 = scale * _mm256_loadu_pd(ps+j+8); |
| 2911 | auto val4 = scale * _mm256_loadu_pd(ps+j+12); |
| 2912 | |
| 2913 | _mm256_store_pd (pd2, val1); |
| 2914 | _mm256_store_pd (pd2+4*distd, val2); |
| 2915 | _mm256_store_pd (pd2+8*distd, val3); |
| 2916 | _mm256_store_pd (pd2+12*distd, val4); |
| 2917 | } |
| 2918 | Prefetch<0> (psnext+j); |
| 2919 | Prefetch<0> (psnext+j+8); |
| 2920 | for ( ; j +4 <= w; j+=4, pd2 += 4*distd) |
| 2921 | _mm256_storeu_pd(pd2, scale * _mm256_loadu_pd(ps+j)); |
| 2922 | _mm256_maskstore_pd (pd2, mask, scale * _mm256_maskload_pd(ps+j, mask)); |
| 2923 | |
| 2924 | ps = psnext; |
| 2925 | } |