| 44 | // |
| 45 | template<typename T> |
| 46 | dim_t vectorizeShape(const dim_t maxVectorWidth, Param<T> &out, dim_t &indims, |
| 47 | CParam<T> &in) { |
| 48 | dim_t vectorWidth{1}; |
| 49 | if ((maxVectorWidth != 1) & (in.strides[0] == 1) & (out.strides[0] == 1)) { |
| 50 | // Only adjacent items can be grouped into a base vector type |
| 51 | void *in_ptr{(void *)in.ptr}; |
| 52 | void *out_ptr{(void *)out.ptr}; |
| 53 | // - global is the OR of the values to be checked. When global is |
| 54 | // divisable by 2, than all source values are also |
| 55 | dim_t global{in.dims[0]}; |
| 56 | for (int i{1}; i < indims; ++i) { |
| 57 | global |= in.strides[i] | out.strides[i]; |
| 58 | } |
| 59 | // - The buffers are always aligned at 128 Bytes. The pointers in the |
| 60 | // Param<T> structure are however, direct pointers (including the |
| 61 | // offset), so the final pointer has to be chedked on alignment |
| 62 | size_t filler{64}; // give enough space for the align to move |
| 63 | unsigned count{0}; |
| 64 | while (((global & 1) == 0) & (vectorWidth < maxVectorWidth) && |
| 65 | (in.ptr == |
| 66 | std::align(alignof(T) * vectorWidth * 2, 1, in_ptr, filler)) && |
| 67 | (out.ptr == |
| 68 | std::align(alignof(T) * vectorWidth * 2, 1, out_ptr, filler))) { |
| 69 | ++count; |
| 70 | vectorWidth <<= 1; |
| 71 | global >>= 1; |
| 72 | } |
| 73 | if (count != 0) { |
| 74 | // update the dimensions, to compensate for the vector base |
| 75 | // type change |
| 76 | in.dims[0] >>= count; |
| 77 | for (int i{1}; i < indims; ++i) { |
| 78 | in.strides[i] >>= count; |
| 79 | out.strides[i] >>= count; |
| 80 | } |
| 81 | if (in.dims[0] == 1) { |
| 82 | // Vectorization has absorbed the full dim0, so eliminate |
| 83 | // this dimension |
| 84 | --indims; |
| 85 | for (int i{0}; i < indims; ++i) { |
| 86 | in.dims[i] = in.dims[i + 1]; |
| 87 | in.strides[i] = in.strides[i + 1]; |
| 88 | out.strides[i] = out.strides[i + 1]; |
| 89 | } |
| 90 | in.dims[indims] = 1; |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | return vectorWidth; |
| 95 | } |
| 96 | |
| 97 | template<typename T> |
| 98 | void memcopy(Param<T> out, CParam<T> in, dim_t indims) { |