| 185 | } |
| 186 | |
| 187 | uint16_t |
| 188 | vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, |
| 189 | struct rte_mbuf **pkts, uint32_t count) |
| 190 | { |
| 191 | struct vhost_queue *queue; |
| 192 | struct rte_vhost_vring *vr; |
| 193 | uint16_t avail_idx, free_entries, start_idx; |
| 194 | uint16_t desc_indexes[MAX_PKT_BURST]; |
| 195 | uint16_t used_idx; |
| 196 | uint32_t i; |
| 197 | |
| 198 | queue = &dev->queues[queue_id]; |
| 199 | vr = &queue->vr; |
| 200 | |
| 201 | avail_idx = __atomic_load_n(&vr->avail->idx, __ATOMIC_ACQUIRE); |
| 202 | start_idx = queue->last_used_idx; |
| 203 | free_entries = avail_idx - start_idx; |
| 204 | count = RTE_MIN(count, free_entries); |
| 205 | count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); |
| 206 | if (count == 0) |
| 207 | return 0; |
| 208 | |
| 209 | /* Retrieve all of the desc indexes first to avoid caching issues. */ |
| 210 | rte_prefetch0(&vr->avail->ring[start_idx & (vr->size - 1)]); |
| 211 | for (i = 0; i < count; i++) { |
| 212 | used_idx = (start_idx + i) & (vr->size - 1); |
| 213 | desc_indexes[i] = vr->avail->ring[used_idx]; |
| 214 | vr->used->ring[used_idx].id = desc_indexes[i]; |
| 215 | vr->used->ring[used_idx].len = pkts[i]->pkt_len + |
| 216 | dev->hdr_len; |
| 217 | } |
| 218 | |
| 219 | rte_prefetch0(&vr->desc[desc_indexes[0]]); |
| 220 | for (i = 0; i < count; i++) { |
| 221 | uint16_t desc_idx = desc_indexes[i]; |
| 222 | int err; |
| 223 | |
| 224 | err = enqueue_pkt(dev, vr, pkts[i], desc_idx); |
| 225 | if (unlikely(err)) { |
| 226 | used_idx = (start_idx + i) & (vr->size - 1); |
| 227 | vr->used->ring[used_idx].len = dev->hdr_len; |
| 228 | } |
| 229 | |
| 230 | if (i + 1 < count) |
| 231 | rte_prefetch0(&vr->desc[desc_indexes[i+1]]); |
| 232 | } |
| 233 | |
| 234 | __atomic_fetch_add(&vr->used->idx, count, __ATOMIC_RELEASE); |
| 235 | queue->last_used_idx += count; |
| 236 | |
| 237 | rte_vhost_vring_call(dev->vid, queue_id); |
| 238 | |
| 239 | return count; |
| 240 | } |
| 241 | |
| 242 | uint16_t |
| 243 | builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, |
no test coverage detected