| 3610 | } |
| 3611 | |
| 3612 | static __rte_always_inline int |
| 3613 | dlb2_recv_qe_sparse(struct dlb2_port *qm_port, struct dlb2_dequeue_qe *qe) |
| 3614 | { |
| 3615 | volatile struct dlb2_dequeue_qe *cq_addr; |
| 3616 | uint8_t xor_mask[2] = {0x0F, 0x00}; |
| 3617 | const uint8_t and_mask = 0x0F; |
| 3618 | __m128i *qes = (__m128i *)qe; |
| 3619 | uint8_t gen_bits, gen_bit; |
| 3620 | uintptr_t addr[4]; |
| 3621 | uint16_t idx; |
| 3622 | |
| 3623 | cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base; |
| 3624 | |
| 3625 | idx = qm_port->cq_idx_unmasked & qm_port->cq_depth_mask; |
| 3626 | /* Load the next 4 QEs */ |
| 3627 | addr[0] = (uintptr_t)&cq_addr[idx]; |
| 3628 | addr[1] = (uintptr_t)&cq_addr[(idx + 4) & qm_port->cq_depth_mask]; |
| 3629 | addr[2] = (uintptr_t)&cq_addr[(idx + 8) & qm_port->cq_depth_mask]; |
| 3630 | addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask]; |
| 3631 | |
| 3632 | /* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */ |
| 3633 | rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]); |
| 3634 | rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]); |
| 3635 | rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]); |
| 3636 | rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]); |
| 3637 | |
| 3638 | /* Correct the xor_mask for wrap-around QEs */ |
| 3639 | gen_bit = qm_port->gen_bit; |
| 3640 | xor_mask[gen_bit] ^= !!((idx + 4) > qm_port->cq_depth_mask) << 1; |
| 3641 | xor_mask[gen_bit] ^= !!((idx + 8) > qm_port->cq_depth_mask) << 2; |
| 3642 | xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3; |
| 3643 | |
| 3644 | /* Read the cache lines backwards to ensure that if QE[N] (N > 0) is |
| 3645 | * valid, then QEs[0:N-1] are too. |
| 3646 | */ |
| 3647 | qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]); |
| 3648 | rte_compiler_barrier(); |
| 3649 | qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]); |
| 3650 | rte_compiler_barrier(); |
| 3651 | qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]); |
| 3652 | rte_compiler_barrier(); |
| 3653 | qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]); |
| 3654 | |
| 3655 | /* Extract and combine the gen bits */ |
| 3656 | gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) | |
| 3657 | ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) | |
| 3658 | ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) | |
| 3659 | ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3); |
| 3660 | |
| 3661 | /* XOR the combined bits such that a 1 represents a valid QE */ |
| 3662 | gen_bits ^= xor_mask[gen_bit]; |
| 3663 | |
| 3664 | /* Mask off gen bits we don't care about */ |
| 3665 | gen_bits &= and_mask; |
| 3666 | |
| 3667 | return rte_popcount32(gen_bits); |
| 3668 | } |
| 3669 |
no test coverage detected