| 3668 | } |
| 3669 | |
| 3670 | static inline void |
| 3671 | _process_deq_qes_vec_impl(struct dlb2_port *qm_port, |
| 3672 | struct rte_event *events, |
| 3673 | __m128i v_qe_3, |
| 3674 | __m128i v_qe_2, |
| 3675 | __m128i v_qe_1, |
| 3676 | __m128i v_qe_0, |
| 3677 | __m128i v_qe_meta, |
| 3678 | __m128i v_qe_status, |
| 3679 | uint32_t valid_events) |
| 3680 | { |
| 3681 | /* Look up the event QIDs, using the hardware QIDs to index the |
| 3682 | * port's QID mapping. |
| 3683 | * |
| 3684 | * Each v_qe_[0-4] is just a 16-byte load of the whole QE. It is |
| 3685 | * passed along in registers as the QE data is required later. |
| 3686 | * |
| 3687 | * v_qe_meta is an u32 unpack of all 4x QEs. A.k.a, it contains one |
| 3688 | * 32-bit slice of each QE, so makes up a full SSE register. This |
| 3689 | * allows parallel processing of 4x QEs in a single register. |
| 3690 | */ |
| 3691 | |
| 3692 | __m128i v_qid_done = {0}; |
| 3693 | int hw_qid0 = _mm_extract_epi8(v_qe_meta, 2); |
| 3694 | int hw_qid1 = _mm_extract_epi8(v_qe_meta, 6); |
| 3695 | int hw_qid2 = _mm_extract_epi8(v_qe_meta, 10); |
| 3696 | int hw_qid3 = _mm_extract_epi8(v_qe_meta, 14); |
| 3697 | |
| 3698 | int ev_qid0 = qm_port->qid_mappings[hw_qid0]; |
| 3699 | int ev_qid1 = qm_port->qid_mappings[hw_qid1]; |
| 3700 | int ev_qid2 = qm_port->qid_mappings[hw_qid2]; |
| 3701 | int ev_qid3 = qm_port->qid_mappings[hw_qid3]; |
| 3702 | |
| 3703 | int hw_sched0 = _mm_extract_epi8(v_qe_meta, 3) & 3ul; |
| 3704 | int hw_sched1 = _mm_extract_epi8(v_qe_meta, 7) & 3ul; |
| 3705 | int hw_sched2 = _mm_extract_epi8(v_qe_meta, 11) & 3ul; |
| 3706 | int hw_sched3 = _mm_extract_epi8(v_qe_meta, 15) & 3ul; |
| 3707 | |
| 3708 | v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid0, 2); |
| 3709 | v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid1, 6); |
| 3710 | v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid2, 10); |
| 3711 | v_qid_done = _mm_insert_epi8(v_qid_done, ev_qid3, 14); |
| 3712 | |
| 3713 | /* Schedule field remapping using byte shuffle |
| 3714 | * - Full byte containing sched field handled here (op, rsvd are zero) |
| 3715 | * - Note sanitizing the register requires two masking ANDs: |
| 3716 | * 1) to strip prio/msg_type from byte for correct shuffle lookup |
| 3717 | * 2) to strip any non-sched-field lanes from any results to OR later |
| 3718 | * - Final byte result is >> 10 to another byte-lane inside the u32. |
| 3719 | * This makes the final combination OR easier to make the rte_event. |
| 3720 | */ |
| 3721 | __m128i v_sched_done; |
| 3722 | __m128i v_sched_bits; |
| 3723 | { |
| 3724 | static const uint8_t sched_type_map[16] = { |
| 3725 | [DLB2_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC, |
| 3726 | [DLB2_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL, |
| 3727 | [DLB2_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED, |
no outgoing calls
no test coverage detected