| 3101 | } |
| 3102 | |
| 3103 | __rte_always_inline |
| 3104 | static uint16_t |
| 3105 | virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 3106 | struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, |
| 3107 | bool legacy_ol_flags) |
| 3108 | __rte_shared_locks_required(&vq->access_lock) |
| 3109 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 3110 | { |
| 3111 | uint16_t i; |
| 3112 | uint16_t avail_entries; |
| 3113 | static bool allocerr_warned; |
| 3114 | |
| 3115 | /* |
| 3116 | * The ordering between avail index and |
| 3117 | * desc reads needs to be enforced. |
| 3118 | */ |
| 3119 | avail_entries = rte_atomic_load_explicit((unsigned short __rte_atomic *)&vq->avail->idx, |
| 3120 | rte_memory_order_acquire) - vq->last_avail_idx; |
| 3121 | if (avail_entries == 0) |
| 3122 | return 0; |
| 3123 | |
| 3124 | rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); |
| 3125 | |
| 3126 | VHOST_LOG_DATA(dev->ifname, DEBUG, "%s\n", __func__); |
| 3127 | |
| 3128 | count = RTE_MIN(count, MAX_PKT_BURST); |
| 3129 | count = RTE_MIN(count, avail_entries); |
| 3130 | VHOST_LOG_DATA(dev->ifname, DEBUG, "about to dequeue %u buffers\n", count); |
| 3131 | |
| 3132 | if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts, count)) |
| 3133 | return 0; |
| 3134 | |
| 3135 | for (i = 0; i < count; i++) { |
| 3136 | struct buf_vector buf_vec[BUF_VECTOR_MAX]; |
| 3137 | uint16_t head_idx; |
| 3138 | uint32_t buf_len; |
| 3139 | uint16_t nr_vec = 0; |
| 3140 | int err; |
| 3141 | |
| 3142 | if (unlikely(fill_vec_buf_split(dev, vq, |
| 3143 | vq->last_avail_idx + i, |
| 3144 | &nr_vec, buf_vec, |
| 3145 | &head_idx, &buf_len, |
| 3146 | VHOST_ACCESS_RO) < 0)) |
| 3147 | break; |
| 3148 | |
| 3149 | update_shadow_used_ring_split(vq, head_idx, 0); |
| 3150 | |
| 3151 | if (unlikely(buf_len <= dev->vhost_hlen)) |
| 3152 | break; |
| 3153 | |
| 3154 | buf_len -= dev->vhost_hlen; |
| 3155 | |
| 3156 | err = virtio_dev_pktmbuf_prep(dev, pkts[i], buf_len); |
| 3157 | if (unlikely(err)) { |
| 3158 | /* |
| 3159 | * mbuf allocation fails for jumbo packets when external |
| 3160 | * buffer allocation is not allowed and linear buffer |
no test coverage detected