| 451 | /// @see MediumKernel for an explanation of the algorithm. |
| 452 | template <typename KerTraits, MediumKernelOptions kOptions> |
| 453 | struct MediumKernelPlan { |
| 454 | using Traits = KerTraits; |
| 455 | using uint_type = typename Traits::uint_type; |
| 456 | static constexpr auto kShape = Traits::kShape; |
| 457 | static constexpr auto kPlanSize = MediumKernelPlanSize::Build(kShape, kOptions); |
| 458 | |
| 459 | /// Array of byte offsets relative to the Kernel input memory address. |
| 460 | using ReadsPerKernel = std::array<int, kPlanSize.reads_per_kernel()>; |
| 461 | |
| 462 | /// Array of indices used for byte swizzle/shuffle in an SIMD register. |
| 463 | using Swizzle = std::array<uint8_t, kShape.simd_byte_size()>; |
| 464 | using SwizzlesPerRead = std::array<Swizzle, kPlanSize.swizzles_per_read()>; |
| 465 | using SwizzlesPerKernel = std::array<SwizzlesPerRead, kPlanSize.reads_per_kernel()>; |
| 466 | |
| 467 | /// Array of integer shifts for each unpacked integer in an SIMD register. |
| 468 | using Shifts = std::array<uint_type, kShape.unpacked_per_simd()>; |
| 469 | using ShiftsPerSwizzle = std::array<Shifts, kPlanSize.shifts_per_swizzle()>; |
| 470 | using ShiftsPerRead = std::array<ShiftsPerSwizzle, kPlanSize.swizzles_per_read()>; |
| 471 | using ShiftsPerKernel = std::array<ShiftsPerRead, kPlanSize.reads_per_kernel()>; |
| 472 | |
| 473 | constexpr static MediumKernelPlan Build(); |
| 474 | |
| 475 | static constexpr int unpacked_per_shift() { return kShape.unpacked_per_simd(); } |
| 476 | static constexpr int unpacked_per_swizzle() { |
| 477 | return unpacked_per_shift() * kPlanSize.shifts_per_swizzle(); |
| 478 | } |
| 479 | static constexpr int unpacked_per_read() { |
| 480 | return unpacked_per_swizzle() * kPlanSize.swizzles_per_read(); |
| 481 | } |
| 482 | static constexpr int unpacked_per_kernel() { |
| 483 | return unpacked_per_read() * kPlanSize.reads_per_kernel(); |
| 484 | } |
| 485 | |
| 486 | static constexpr int bytes_per_read() { |
| 487 | const auto bits_per_read = unpacked_per_read() * kShape.packed_bit_size(); |
| 488 | return adjust_bytes_per_read(bits_per_read, kShape.simd_byte_size()); |
| 489 | } |
| 490 | |
| 491 | constexpr int total_bytes_read() const { return reads.back() + bytes_per_read(); } |
| 492 | |
| 493 | ReadsPerKernel reads; |
| 494 | SwizzlesPerKernel swizzles; |
| 495 | ShiftsPerKernel shifts; |
| 496 | uint_type mask = bit_util::LeastSignificantBitMask<uint_type>(kShape.packed_bit_size()); |
| 497 | }; |
| 498 | |
| 499 | template <typename KerTraits, MediumKernelOptions kOptions> |
| 500 | constexpr auto MediumKernelPlan<KerTraits, kOptions>::Build() |
nothing calls this directly
no test coverage detected