| 3216 | } |
| 3217 | |
| 3218 | static __rte_always_inline int |
| 3219 | vhost_reserve_avail_batch_packed(struct virtio_net *dev, |
| 3220 | struct vhost_virtqueue *vq, |
| 3221 | struct rte_mbuf **pkts, |
| 3222 | uint16_t avail_idx, |
| 3223 | uintptr_t *desc_addrs, |
| 3224 | uint16_t *ids) |
| 3225 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 3226 | { |
| 3227 | bool wrap = vq->avail_wrap_counter; |
| 3228 | struct vring_packed_desc *descs = vq->desc_packed; |
| 3229 | uint64_t lens[PACKED_BATCH_SIZE]; |
| 3230 | uint64_t buf_lens[PACKED_BATCH_SIZE]; |
| 3231 | uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf); |
| 3232 | uint16_t flags, i; |
| 3233 | |
| 3234 | if (unlikely(avail_idx & PACKED_BATCH_MASK)) |
| 3235 | return -1; |
| 3236 | if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size)) |
| 3237 | return -1; |
| 3238 | |
| 3239 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3240 | flags = descs[avail_idx + i].flags; |
| 3241 | if (unlikely((wrap != !!(flags & VRING_DESC_F_AVAIL)) || |
| 3242 | (wrap == !!(flags & VRING_DESC_F_USED)) || |
| 3243 | (flags & PACKED_DESC_SINGLE_DEQUEUE_FLAG))) |
| 3244 | return -1; |
| 3245 | } |
| 3246 | |
| 3247 | rte_atomic_thread_fence(rte_memory_order_acquire); |
| 3248 | |
| 3249 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) |
| 3250 | lens[i] = descs[avail_idx + i].len; |
| 3251 | |
| 3252 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3253 | desc_addrs[i] = vhost_iova_to_vva(dev, vq, |
| 3254 | descs[avail_idx + i].addr, |
| 3255 | &lens[i], VHOST_ACCESS_RW); |
| 3256 | } |
| 3257 | |
| 3258 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3259 | if (unlikely(!desc_addrs[i])) |
| 3260 | return -1; |
| 3261 | if (unlikely((lens[i] != descs[avail_idx + i].len))) |
| 3262 | return -1; |
| 3263 | } |
| 3264 | |
| 3265 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3266 | if (virtio_dev_pktmbuf_prep(dev, pkts[i], lens[i])) |
| 3267 | goto err; |
| 3268 | } |
| 3269 | |
| 3270 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) |
| 3271 | buf_lens[i] = pkts[i]->buf_len - pkts[i]->data_off; |
| 3272 | |
| 3273 | vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { |
| 3274 | if (unlikely(buf_lens[i] < (lens[i] - buf_offset))) |
| 3275 | goto err; |
no test coverage detected