| 48 | return args.layout_a.shape[0] * 3 * sizeof(uintptr_t); |
| 49 | } |
| 50 | void BatchedMatrixMulForwardImpl::AlgoCublas::exec(const ExecArgs& args) const { |
| 51 | auto param = args.opr->param(); |
| 52 | auto dtype = args.layout_a.dtype; |
| 53 | auto handle = concrete_handle(args.opr->handle()); |
| 54 | auto cublas_handle = handle->cublas_handle(); |
| 55 | auto stream = cuda_stream(handle); |
| 56 | auto batch = args.layout_a.shape[0]; |
| 57 | auto m = args.layout_c.shape[1], n = args.layout_c.shape[2]; |
| 58 | auto k = args.layout_a.shape[param.transposeA ? 1 : 2]; |
| 59 | auto workspace = args.workspace; |
| 60 | |
| 61 | uintptr_t* As = static_cast<uintptr_t*>( |
| 62 | static_cast<void*>(workspace.raw_ptr + 0 * batch * sizeof(uintptr_t))); |
| 63 | uintptr_t* Bs = static_cast<uintptr_t*>( |
| 64 | static_cast<void*>(workspace.raw_ptr + 1 * batch * sizeof(uintptr_t))); |
| 65 | uintptr_t* Cs = static_cast<uintptr_t*>( |
| 66 | static_cast<void*>(workspace.raw_ptr + 2 * batch * sizeof(uintptr_t))); |
| 67 | |
| 68 | arange<uintptr_t>( |
| 69 | As, reinterpret_cast<uintptr_t>(args.tensor_a.raw_ptr()), |
| 70 | args.layout_a.stride[0] * dtype.size(), batch, stream); |
| 71 | arange<uintptr_t>( |
| 72 | Bs, reinterpret_cast<uintptr_t>(args.tensor_b.raw_ptr()), |
| 73 | args.layout_b.stride[0] * dtype.size(), batch, stream); |
| 74 | arange<uintptr_t>( |
| 75 | Cs, reinterpret_cast<uintptr_t>(args.tensor_c.raw_ptr()), |
| 76 | args.layout_c.stride[0] * dtype.size(), batch, stream); |
| 77 | |
| 78 | auto io32_c32 = [&]() { |
| 79 | auto zero = handle->zero_device(); |
| 80 | auto one = handle->one_device(); |
| 81 | cublas_check(cublasSgemmBatched( |
| 82 | cublas_handle, param.transposeB ? CUBLAS_OP_T : CUBLAS_OP_N, |
| 83 | param.transposeA ? CUBLAS_OP_T : CUBLAS_OP_N, n, m, k, one, |
| 84 | reinterpret_cast<const dt_float32**>(Bs), args.layout_b.stride[1], |
| 85 | reinterpret_cast<const dt_float32**>(As), args.layout_a.stride[1], zero, |
| 86 | reinterpret_cast<dt_float32**>(Cs), args.layout_c.stride[1], batch)); |
| 87 | }; |
| 88 | |
| 89 | #if CUDART_VERSION >= 9010 |
| 90 | auto io16_c32 = [&]() { |
| 91 | #if CUDART_VERSION >= 11000 |
| 92 | cublas_check(cublasSetMathMode(cublas_handle, CUBLAS_TF32_TENSOR_OP_MATH)); |
| 93 | #else |
| 94 | cublas_check(cublasSetMathMode(cublas_handle, CUBLAS_TENSOR_OP_MATH)); |
| 95 | #endif |
| 96 | auto zero = handle->zero_device(); |
| 97 | auto one = handle->one_device(); |
| 98 | cublas_check(cublasGemmBatchedEx( |
| 99 | cublas_handle, param.transposeB ? CUBLAS_OP_T : CUBLAS_OP_N, |
| 100 | param.transposeA ? CUBLAS_OP_T : CUBLAS_OP_N, n, m, k, one, |
| 101 | reinterpret_cast<const void**>(Bs), CUDA_R_16F, args.layout_b.stride[1], |
| 102 | reinterpret_cast<const void**>(As), CUDA_R_16F, args.layout_a.stride[1], |
| 103 | zero, reinterpret_cast<void**>(Cs), CUDA_R_16F, args.layout_c.stride[1], |
| 104 | batch, CUDA_R_32F, CUBLAS_GEMM_DEFAULT)); |
| 105 | cublas_check(cublasSetMathMode(cublas_handle, CUBLAS_DEFAULT_MATH)); |
| 106 | }; |
| 107 | #endif |
nothing calls this directly
no test coverage detected