| 3859 | } |
| 3860 | |
| 3861 | static __rte_always_inline int |
| 3862 | dlb2_recv_qe_sparse_vec(struct dlb2_port *qm_port, void *events, |
| 3863 | uint32_t max_events) |
| 3864 | { |
| 3865 | /* Using unmasked idx for perf, and masking manually */ |
| 3866 | uint16_t idx = qm_port->cq_idx_unmasked; |
| 3867 | volatile struct dlb2_dequeue_qe *cq_addr; |
| 3868 | |
| 3869 | cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base; |
| 3870 | |
| 3871 | uintptr_t qe_ptr_3 = (uintptr_t)&cq_addr[(idx + 12) & |
| 3872 | qm_port->cq_depth_mask]; |
| 3873 | uintptr_t qe_ptr_2 = (uintptr_t)&cq_addr[(idx + 8) & |
| 3874 | qm_port->cq_depth_mask]; |
| 3875 | uintptr_t qe_ptr_1 = (uintptr_t)&cq_addr[(idx + 4) & |
| 3876 | qm_port->cq_depth_mask]; |
| 3877 | uintptr_t qe_ptr_0 = (uintptr_t)&cq_addr[(idx + 0) & |
| 3878 | qm_port->cq_depth_mask]; |
| 3879 | |
| 3880 | /* Load QEs from CQ: use compiler barriers to avoid load reordering */ |
| 3881 | __m128i v_qe_3 = _mm_loadu_si128((const __m128i *)qe_ptr_3); |
| 3882 | rte_compiler_barrier(); |
| 3883 | __m128i v_qe_2 = _mm_loadu_si128((const __m128i *)qe_ptr_2); |
| 3884 | rte_compiler_barrier(); |
| 3885 | __m128i v_qe_1 = _mm_loadu_si128((const __m128i *)qe_ptr_1); |
| 3886 | rte_compiler_barrier(); |
| 3887 | __m128i v_qe_0 = _mm_loadu_si128((const __m128i *)qe_ptr_0); |
| 3888 | |
| 3889 | /* Generate the pkt_shuffle mask; |
| 3890 | * - Avoids load in otherwise load-heavy section of code |
| 3891 | * - Moves bytes 3,7,11,15 (gen bit bytes) to LSB bytes in XMM |
| 3892 | */ |
| 3893 | const uint32_t stat_shuf_bytes = (15 << 24) | (11 << 16) | (7 << 8) | 3; |
| 3894 | __m128i v_zeros = _mm_setzero_si128(); |
| 3895 | __m128i v_ffff = _mm_cmpeq_epi8(v_zeros, v_zeros); |
| 3896 | __m128i v_stat_shuf_mask = _mm_insert_epi32(v_ffff, stat_shuf_bytes, 0); |
| 3897 | |
| 3898 | /* Extract u32 components required from the QE |
| 3899 | * - QE[64 to 95 ] for metadata (qid, sched, prio, event type, ...) |
| 3900 | * - QE[96 to 127] for status (cq gen bit, error) |
| 3901 | * |
| 3902 | * Note that stage 1 of the unpacking is re-used for both u32 extracts |
| 3903 | */ |
| 3904 | __m128i v_qe_02 = _mm_unpackhi_epi32(v_qe_0, v_qe_2); |
| 3905 | __m128i v_qe_13 = _mm_unpackhi_epi32(v_qe_1, v_qe_3); |
| 3906 | __m128i v_qe_status = _mm_unpackhi_epi32(v_qe_02, v_qe_13); |
| 3907 | __m128i v_qe_meta = _mm_unpacklo_epi32(v_qe_02, v_qe_13); |
| 3908 | |
| 3909 | /* Status byte (gen_bit, error) handling: |
| 3910 | * - Shuffle to lanes 0,1,2,3, clear all others |
| 3911 | * - Shift right by 7 for gen bit to MSB, movemask to scalar |
| 3912 | * - Shift right by 2 for error bit to MSB, movemask to scalar |
| 3913 | */ |
| 3914 | __m128i v_qe_shuffled = _mm_shuffle_epi8(v_qe_status, v_stat_shuf_mask); |
| 3915 | __m128i v_qes_shift_gen_bit = _mm_slli_epi32(v_qe_shuffled, 7); |
| 3916 | int32_t qe_gen_bits = _mm_movemask_epi8(v_qes_shift_gen_bit) & 0xf; |
| 3917 | |
| 3918 | /* Expected vs Reality of QE Gen bits |
no test coverage detected