| 4055 | } |
| 4056 | |
| 4057 | static __rte_always_inline int |
| 4058 | dlb2_recv_qe(struct dlb2_port *qm_port, struct dlb2_dequeue_qe *qe, |
| 4059 | uint8_t *offset) |
| 4060 | { |
| 4061 | uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08}, |
| 4062 | {0x00, 0x01, 0x03, 0x07} }; |
| 4063 | uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08}; |
| 4064 | volatile struct dlb2_dequeue_qe *cq_addr; |
| 4065 | __m128i *qes = (__m128i *)qe; |
| 4066 | uint64_t *cache_line_base; |
| 4067 | uint8_t gen_bits; |
| 4068 | |
| 4069 | cq_addr = dlb2_port[qm_port->id][PORT_TYPE(qm_port)].cq_base; |
| 4070 | cq_addr = &cq_addr[qm_port->cq_idx]; |
| 4071 | |
| 4072 | cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F); |
| 4073 | *offset = ((uintptr_t)cq_addr & 0x30) >> 4; |
| 4074 | |
| 4075 | /* Load the next CQ cache line from memory. Pack these reads as tight |
| 4076 | * as possible to reduce the chance that DLB invalidates the line while |
| 4077 | * the CPU is reading it. Read the cache line backwards to ensure that |
| 4078 | * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too. |
| 4079 | * |
| 4080 | * (Valid QEs start at &qe[offset]) |
| 4081 | */ |
| 4082 | qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]); |
| 4083 | qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]); |
| 4084 | qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]); |
| 4085 | qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]); |
| 4086 | |
| 4087 | /* Evict the cache line ASAP */ |
| 4088 | rte_cldemote(cache_line_base); |
| 4089 | |
| 4090 | /* Extract and combine the gen bits */ |
| 4091 | gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) | |
| 4092 | ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) | |
| 4093 | ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) | |
| 4094 | ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3); |
| 4095 | |
| 4096 | /* XOR the combined bits such that a 1 represents a valid QE */ |
| 4097 | gen_bits ^= xor_mask[qm_port->gen_bit][*offset]; |
| 4098 | |
| 4099 | /* Mask off gen bits we don't care about */ |
| 4100 | gen_bits &= and_mask[*offset]; |
| 4101 | |
| 4102 | return rte_popcount32(gen_bits); |
| 4103 | } |
| 4104 | |
| 4105 | static inline int16_t |
| 4106 | dlb2_hw_dequeue(struct dlb2_eventdev *dlb2, |
no test coverage detected