transpose matrix of avx-vectors (i.e. don't shuffle)
| 2779 | |
| 2780 | // transpose matrix of avx-vectors (i.e. don't shuffle) |
| 2781 | void CopyMatrixInVTrans (size_t h, size_t w, |
| 2782 | double * ps, size_t dists, |
| 2783 | double * pd, size_t distd) |
| 2784 | { |
| 2785 | __m256i mask = my_mm256_cmpgt_epi64(_mm256_set1_epi64x(w&3), |
| 2786 | _mm256_set_epi64x(3,2,1,0)); |
| 2787 | size_t i = 0; |
| 2788 | |
| 2789 | /* |
| 2790 | for ( ; i+3 < h; i+=4, pd += 16) |
| 2791 | { |
| 2792 | double * psnext = ps+4*dists; |
| 2793 | size_t j = 0; |
| 2794 | double * pd2 = pd; |
| 2795 | |
| 2796 | for ( ; j +4 <= w; j+=4, pd2 += 4*distd) |
| 2797 | { |
| 2798 | // Prefetch (ps+4*dists+j, _MM_HINT_T1); |
| 2799 | // Prefetch (ps+5*dists+j, _MM_HINT_T1); |
| 2800 | // Prefetch (ps+6*dists+j, _MM_HINT_T1); |
| 2801 | // Prefetch (ps+7*dists+j, _MM_HINT_T1); |
| 2802 | _mm256_store_pd(pd2, _mm256_loadu_pd(ps+j)); |
| 2803 | _mm256_store_pd(pd2+4, _mm256_loadu_pd(ps+dists+j)); |
| 2804 | _mm256_store_pd(pd2+8, _mm256_loadu_pd(ps+2*dists+j)); |
| 2805 | _mm256_store_pd(pd2+12, _mm256_loadu_pd(ps+3*dists+j)); |
| 2806 | } |
| 2807 | |
| 2808 | _mm256_maskstore_pd (pd2, mask, _mm256_maskload_pd(ps+j, mask)); |
| 2809 | _mm256_maskstore_pd (pd2+4, mask, _mm256_maskload_pd(ps+dists+j, mask)); |
| 2810 | ps = psnext; |
| 2811 | } |
| 2812 | */ |
| 2813 | |
| 2814 | for ( ; i+1 < h; i+=2, pd += 8) |
| 2815 | { |
| 2816 | double * psnext = ps+2*dists; |
| 2817 | size_t j = 0; |
| 2818 | double * pd2 = pd; |
| 2819 | |
| 2820 | for ( ; j+4 <= w; j+=4, pd2 += 4*distd) |
| 2821 | { |
| 2822 | // Prefetch (psnext+j, _MM_HINT_T0); |
| 2823 | // Prefetch (psnext+dists+j, _MM_HINT_T0); |
| 2824 | _mm256_store_pd(pd2, _mm256_loadu_pd(ps+j)); |
| 2825 | _mm256_store_pd(pd2+4, _mm256_loadu_pd(ps+dists+j)); |
| 2826 | } |
| 2827 | |
| 2828 | _mm256_maskstore_pd (pd2, mask, _mm256_maskload_pd(ps+j, mask)); |
| 2829 | _mm256_maskstore_pd (pd2+4, mask, _mm256_maskload_pd(ps+dists+j, mask)); |
| 2830 | ps = psnext; |
| 2831 | } |
| 2832 | |
| 2833 | for ( /* size_t i = 0 */; i < h; i++, pd += 4) |
| 2834 | { |
| 2835 | double * psnext = ps+dists; |
| 2836 | size_t j = 0; |
| 2837 | double * pd2 = pd; |
| 2838 | /* |
no outgoing calls
no test coverage detected