| 2280 | } |
| 2281 | |
| 2282 | static __rte_always_inline uint16_t |
| 2283 | vhost_poll_enqueue_completed(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 2284 | struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id) |
| 2285 | __rte_shared_locks_required(&vq->access_lock) |
| 2286 | { |
| 2287 | struct vhost_async *async = vq->async; |
| 2288 | struct async_inflight_info *pkts_info = async->pkts_info; |
| 2289 | uint16_t nr_cpl_pkts = 0; |
| 2290 | uint16_t n_descs = 0, n_buffers = 0; |
| 2291 | uint16_t start_idx, from, i; |
| 2292 | |
| 2293 | /* Check completed copies for the given DMA vChannel */ |
| 2294 | vhost_async_dma_check_completed(dev, dma_id, vchan_id, VHOST_DMA_MAX_COPY_COMPLETE); |
| 2295 | |
| 2296 | start_idx = async_get_first_inflight_pkt_idx(vq); |
| 2297 | /** |
| 2298 | * Calculate the number of copy completed packets. |
| 2299 | * Note that there may be completed packets even if |
| 2300 | * no copies are reported done by the given DMA vChannel, |
| 2301 | * as it's possible that a virtqueue uses multiple DMA |
| 2302 | * vChannels. |
| 2303 | */ |
| 2304 | from = start_idx; |
| 2305 | while (vq->async->pkts_cmpl_flag[from] && count--) { |
| 2306 | vq->async->pkts_cmpl_flag[from] = false; |
| 2307 | from++; |
| 2308 | if (from >= vq->size) |
| 2309 | from -= vq->size; |
| 2310 | nr_cpl_pkts++; |
| 2311 | } |
| 2312 | |
| 2313 | if (nr_cpl_pkts == 0) |
| 2314 | return 0; |
| 2315 | |
| 2316 | for (i = 0; i < nr_cpl_pkts; i++) { |
| 2317 | from = (start_idx + i) % vq->size; |
| 2318 | /* Only used with packed ring */ |
| 2319 | n_buffers += pkts_info[from].nr_buffers; |
| 2320 | /* Only used with split ring */ |
| 2321 | n_descs += pkts_info[from].descs; |
| 2322 | pkts[i] = pkts_info[from].mbuf; |
| 2323 | } |
| 2324 | |
| 2325 | async->pkts_inflight_n -= nr_cpl_pkts; |
| 2326 | |
| 2327 | if (likely(vq->enabled && vq->access_ok)) { |
| 2328 | if (vq_is_packed(dev)) { |
| 2329 | write_back_completed_descs_packed(vq, n_buffers); |
| 2330 | vhost_vring_call_packed(dev, vq); |
| 2331 | } else { |
| 2332 | write_back_completed_descs_split(vq, n_descs); |
| 2333 | rte_atomic_fetch_add_explicit( |
| 2334 | (unsigned short __rte_atomic *)&vq->used->idx, |
| 2335 | n_descs, rte_memory_order_release); |
| 2336 | vhost_vring_call_split(dev, vq); |
| 2337 | } |
| 2338 | } else { |
| 2339 | if (vq_is_packed(dev)) { |
no test coverage detected