| 639 | |
| 640 | template <typename TypeInput, typename TypeWeight, typename TypeOutput, class OutputStage> |
| 641 | void Fallback<TypeInput, TypeWeight, TypeOutput, OutputStage>::run(ITensorPack &tensors) |
| 642 | { |
| 643 | auto a = tensors.get_const_tensor(TensorType::ACL_SRC_0); |
| 644 | auto b = tensors.get_const_tensor(TensorType::ACL_SRC_1); |
| 645 | auto c = tensors.get_const_tensor(TensorType::ACL_SRC_2); |
| 646 | auto d = tensors.get_tensor(TensorType::ACL_DST); |
| 647 | ARM_COMPUTE_ERROR_ON_NULLPTR(a, d); |
| 648 | |
| 649 | // Only update at runtime if the src quantization is dynamic |
| 650 | if (std::is_same<OutputStage, arm_gemm::DequantizeFloat>::value && |
| 651 | (a->info()->quantization_info().is_dynamic() || b->info()->quantization_info().is_dynamic())) |
| 652 | { |
| 653 | // Output dequantization is just the two src scales multiplied together |
| 654 | _gemm_kernel_asm->set_dequantize_scale(a->info()->quantization_info().uniform().scale * |
| 655 | b->info()->quantization_info().uniform().scale); |
| 656 | } |
| 657 | |
| 658 | int lda = a->info()->strides_in_bytes().y() / a->info()->element_size(); |
| 659 | int ldb = 0; |
| 660 | const int ldd = d->info()->strides_in_bytes().y() / d->info()->element_size(); |
| 661 | |
| 662 | const size_t a_batch_idx = _gemm_info.reinterpret_input_as_3d != 0 ? 3 : 2; |
| 663 | const size_t a_multi_idx = a_batch_idx + 1; |
| 664 | const size_t d_batch_idx = _gemm_info.depth_output_gemm3d != 0 ? 3 : 2; |
| 665 | const size_t d_multi_idx = d_batch_idx + 1; |
| 666 | |
| 667 | int batch_stride_a = a->info()->strides_in_bytes()[a_batch_idx] / a->info()->element_size(); |
| 668 | const int batch_stride_d = d->info()->strides_in_bytes()[d_batch_idx] / d->info()->element_size(); |
| 669 | |
| 670 | int multi_stride_a = a->info()->strides_in_bytes()[a_multi_idx] / a->info()->element_size(); |
| 671 | int multi_stride_b = 0; |
| 672 | const int multi_stride_d = d->info()->strides_in_bytes()[d_multi_idx] / d->info()->element_size(); |
| 673 | |
| 674 | auto in0_ptr = reinterpret_cast<const TypeInput *>(a->buffer() + a->info()->offset_first_element_in_bytes()); |
| 675 | const TypeWeight *in1_ptr = nullptr; |
| 676 | auto out_ptr = reinterpret_cast<TypeOutput *>(d->buffer() + d->info()->offset_first_element_in_bytes()); |
| 677 | |
| 678 | const ITensor *b_to_use = b; |
| 679 | |
| 680 | // Pre-pretranspose B if required |
| 681 | CpuAuxTensorHandler pre_pretransposed_b( |
| 682 | offset_int_vec(PrePretransposedB), _pre_pretransposed_b_info, tensors, |
| 683 | false /*pack_inject: no need to inject into tensors*/, |
| 684 | !_run_pre_pretranspose_b /*bypass_alloc: no need to allocate if pre-pretranspose B is not required as this handle will not be used*/); |
| 685 | if (b_to_use && !_is_b_constant && _run_pre_pretranspose_b) |
| 686 | { |
| 687 | ARM_COMPUTE_ERROR_ON(_pre_pretranspose_b == nullptr); |
| 688 | ITensorPack pre_pretranspose_pack{{ACL_SRC, b_to_use}, {ACL_DST, pre_pretransposed_b.get()}}; |
| 689 | _pre_pretranspose_b->run(pre_pretranspose_pack); |
| 690 | b_to_use = pre_pretransposed_b.get(); |
| 691 | } |
| 692 | |
| 693 | // Check if B is pre-tranposed and de-reference if not |
| 694 | if (b_to_use && !_gemm_kernel_asm->B_is_pretransposed()) |
| 695 | { |
| 696 | ldb = b_to_use->info()->strides_in_bytes().y() / b_to_use->info()->element_size(); |
| 697 | multi_stride_b = b_to_use->info()->strides_in_bytes().z() / b_to_use->info()->element_size(); |
| 698 | in1_ptr = reinterpret_cast<const TypeWeight *>(b_to_use->buffer() + |
no test coverage detected