| 1402 | } |
| 1403 | |
| 1404 | static __rte_noinline uint32_t |
| 1405 | virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 1406 | struct rte_mbuf **pkts, uint32_t count) |
| 1407 | __rte_shared_locks_required(&vq->access_lock) |
| 1408 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 1409 | { |
| 1410 | uint32_t pkt_idx = 0; |
| 1411 | uint16_t num_buffers; |
| 1412 | struct buf_vector buf_vec[BUF_VECTOR_MAX]; |
| 1413 | uint16_t avail_head; |
| 1414 | |
| 1415 | /* |
| 1416 | * The ordering between avail index and |
| 1417 | * desc reads needs to be enforced. |
| 1418 | */ |
| 1419 | avail_head = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, |
| 1420 | rte_memory_order_acquire); |
| 1421 | |
| 1422 | rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); |
| 1423 | |
| 1424 | for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { |
| 1425 | uint64_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; |
| 1426 | uint16_t nr_vec = 0; |
| 1427 | |
| 1428 | if (unlikely(reserve_avail_buf_split(dev, vq, |
| 1429 | pkt_len, buf_vec, &num_buffers, |
| 1430 | avail_head, &nr_vec) < 0)) { |
| 1431 | VHOST_LOG_DATA(dev->ifname, DEBUG, |
| 1432 | "failed to get enough desc from vring\n"); |
| 1433 | vq->shadow_used_idx -= num_buffers; |
| 1434 | break; |
| 1435 | } |
| 1436 | |
| 1437 | VHOST_LOG_DATA(dev->ifname, DEBUG, |
| 1438 | "current index %d | end index %d\n", |
| 1439 | vq->last_avail_idx, vq->last_avail_idx + num_buffers); |
| 1440 | |
| 1441 | if (mbuf_to_desc(dev, vq, pkts[pkt_idx], buf_vec, nr_vec, |
| 1442 | num_buffers, false) < 0) { |
| 1443 | vq->shadow_used_idx -= num_buffers; |
| 1444 | break; |
| 1445 | } |
| 1446 | |
| 1447 | vq->last_avail_idx += num_buffers; |
| 1448 | } |
| 1449 | |
| 1450 | do_data_copy_enqueue(dev, vq); |
| 1451 | |
| 1452 | if (likely(vq->shadow_used_idx)) { |
| 1453 | flush_shadow_used_ring_split(dev, vq); |
| 1454 | vhost_vring_call_split(dev, vq); |
| 1455 | } |
| 1456 | |
| 1457 | return pkt_idx; |
| 1458 | } |
| 1459 | |
| 1460 | static __rte_always_inline int |
| 1461 | virtio_dev_rx_sync_batch_check(struct virtio_net *dev, |
no test coverage detected