| 3362 | } |
| 3363 | |
| 3364 | static __rte_always_inline int |
| 3365 | virtio_dev_tx_batch_packed(struct virtio_net *dev, |
| 3366 | struct vhost_virtqueue *vq, |
| 3367 | struct rte_mbuf **pkts, |
| 3368 | bool legacy_ol_flags) |
| 3369 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 3370 | { |
| 3371 | uint16_t avail_idx = vq->last_avail_idx; |
| 3372 | uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf); |
| 3373 | struct virtio_net_hdr *hdr; |
| 3374 | uintptr_t desc_addrs[PACKED_BATCH_SIZE]; |
| 3375 | uint16_t ids[PACKED_BATCH_SIZE]; |
| 3376 | uint16_t i; |
| 3377 | |
| 3378 | if (vhost_reserve_avail_batch_packed(dev, vq, pkts, avail_idx, |
| 3379 | desc_addrs, ids)) |
| 3380 | return -1; |
| 3381 | |
| 3382 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) |
| 3383 | rte_prefetch0((void *)(uintptr_t)desc_addrs[i]); |
| 3384 | |
| 3385 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) |
| 3386 | rte_memcpy(rte_pktmbuf_mtod_offset(pkts[i], void *, 0), |
| 3387 | (void *)(uintptr_t)(desc_addrs[i] + buf_offset), |
| 3388 | pkts[i]->pkt_len); |
| 3389 | |
| 3390 | if (virtio_net_with_host_offload(dev)) { |
| 3391 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3392 | hdr = (struct virtio_net_hdr *)(desc_addrs[i]); |
| 3393 | vhost_dequeue_offload(dev, hdr, pkts[i], legacy_ol_flags); |
| 3394 | } |
| 3395 | } |
| 3396 | |
| 3397 | if (virtio_net_is_inorder(dev)) |
| 3398 | vhost_shadow_dequeue_batch_packed_inorder(vq, |
| 3399 | ids[PACKED_BATCH_SIZE - 1]); |
| 3400 | else |
| 3401 | vhost_shadow_dequeue_batch_packed(dev, vq, ids); |
| 3402 | |
| 3403 | vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE); |
| 3404 | |
| 3405 | return 0; |
| 3406 | } |
| 3407 | |
| 3408 | static __rte_always_inline int |
| 3409 | vhost_dequeue_single_packed(struct virtio_net *dev, |
no test coverage detected