| 276 | } |
| 277 | |
| 278 | static __rte_always_inline void |
| 279 | flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq) |
| 280 | { |
| 281 | uint16_t used_idx = vq->last_used_idx & (vq->size - 1); |
| 282 | |
| 283 | if (used_idx + vq->shadow_used_idx <= vq->size) { |
| 284 | do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, |
| 285 | vq->shadow_used_idx); |
| 286 | } else { |
| 287 | uint16_t size; |
| 288 | |
| 289 | /* update used ring interval [used_idx, vq->size] */ |
| 290 | size = vq->size - used_idx; |
| 291 | do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size); |
| 292 | |
| 293 | /* update the left half used ring interval [0, left_size] */ |
| 294 | do_flush_shadow_used_ring_split(dev, vq, 0, size, |
| 295 | vq->shadow_used_idx - size); |
| 296 | } |
| 297 | vq->last_used_idx += vq->shadow_used_idx; |
| 298 | |
| 299 | vhost_log_cache_sync(dev, vq); |
| 300 | |
| 301 | rte_atomic_fetch_add_explicit((unsigned short __rte_atomic *)&vq->used->idx, |
| 302 | vq->shadow_used_idx, rte_memory_order_release); |
| 303 | vq->shadow_used_idx = 0; |
| 304 | vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), |
| 305 | sizeof(vq->used->idx)); |
| 306 | } |
| 307 | |
| 308 | static __rte_always_inline void |
| 309 | update_shadow_used_ring_split(struct vhost_virtqueue *vq, |
no test coverage detected