| 64 | |
| 65 | template<typename Ti, typename To> |
| 66 | void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs, |
| 67 | const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs, |
| 68 | const To *beta) { |
| 69 | #if defined(WITH_LINEAR_ALGEBRA) |
| 70 | // Do not force offload gemm on OSX Intel devices |
| 71 | if (OpenCLCPUOffload(false) && |
| 72 | static_cast<af_dtype>(dtype_traits<Ti>::af_type) != f16) { |
| 73 | gemm_fallback(out, optLhs, optRhs, alpha, lhs, rhs, beta); |
| 74 | return; |
| 75 | } |
| 76 | #endif |
| 77 | const auto lOpts = toBlasTranspose(optLhs); |
| 78 | const auto rOpts = toBlasTranspose(optRhs); |
| 79 | |
| 80 | const auto aRowDim = (lOpts == OPENCL_BLAS_NO_TRANS) ? 0 : 1; |
| 81 | const auto aColDim = (lOpts == OPENCL_BLAS_NO_TRANS) ? 1 : 0; |
| 82 | const auto bColDim = (rOpts == OPENCL_BLAS_NO_TRANS) ? 1 : 0; |
| 83 | |
| 84 | const dim4 &lDims = lhs.dims(); |
| 85 | const dim4 &rDims = rhs.dims(); |
| 86 | const int M = lDims[aRowDim]; |
| 87 | const int N = rDims[bColDim]; |
| 88 | const int K = lDims[aColDim]; |
| 89 | const dim4 oDims = out.dims(); |
| 90 | |
| 91 | const dim4 &lStrides = lhs.strides(); |
| 92 | const dim4 &rStrides = rhs.strides(); |
| 93 | const dim4 oStrides = out.strides(); |
| 94 | |
| 95 | int batchSize = static_cast<int>(oDims[2] * oDims[3]); |
| 96 | |
| 97 | bool is_l_d2_batched = oDims[2] == lDims[2]; |
| 98 | bool is_l_d3_batched = oDims[3] == lDims[3]; |
| 99 | bool is_r_d2_batched = oDims[2] == rDims[2]; |
| 100 | bool is_r_d3_batched = oDims[3] == rDims[3]; |
| 101 | |
| 102 | for (int n = 0; n < batchSize; n++) { |
| 103 | int w = static_cast<int>(n / oDims[2]); |
| 104 | int z = static_cast<int>(n - w * oDims[2]); |
| 105 | |
| 106 | int loff = z * (is_l_d2_batched * lStrides[2]) + |
| 107 | w * (is_l_d3_batched * lStrides[3]); |
| 108 | int roff = z * (is_r_d2_batched * rStrides[2]) + |
| 109 | w * (is_r_d3_batched * rStrides[3]); |
| 110 | |
| 111 | dim_t lOffset = lhs.getOffset() + loff; |
| 112 | dim_t rOffset = rhs.getOffset() + roff; |
| 113 | dim_t oOffset = out.getOffset() + z * oStrides[2] + w * oStrides[3]; |
| 114 | |
| 115 | cl::Event event; |
| 116 | if (rDims[bColDim] == 1) { |
| 117 | dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; |
| 118 | gpu_blas_gemv_func<Ti> gemv; |
| 119 | OPENCL_BLAS_CHECK(gemv(lOpts, lDims[0], lDims[1], *alpha, |
| 120 | (*lhs.get())(), lOffset, lStrides[1], |
| 121 | (*rhs.get())(), rOffset, incr, *beta, |
| 122 | (*out.get())(), oOffset, oStrides[0], 1, |
| 123 | &getQueue()(), 0, nullptr, &event())); |
no test coverage detected