| 978 | } |
| 979 | |
| 980 | static __rte_always_inline void |
| 981 | vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) |
| 982 | { |
| 983 | uint16_t old, new, off, off_wrap; |
| 984 | bool signalled_used_valid, kick = false; |
| 985 | |
| 986 | /* Flush used desc update. */ |
| 987 | rte_atomic_thread_fence(rte_memory_order_seq_cst); |
| 988 | |
| 989 | if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { |
| 990 | if (vq->driver_event->flags != |
| 991 | VRING_EVENT_F_DISABLE) |
| 992 | kick = true; |
| 993 | goto kick; |
| 994 | } |
| 995 | |
| 996 | old = vq->signalled_used; |
| 997 | new = vq->last_used_idx; |
| 998 | vq->signalled_used = new; |
| 999 | signalled_used_valid = vq->signalled_used_valid; |
| 1000 | vq->signalled_used_valid = true; |
| 1001 | |
| 1002 | if (vq->driver_event->flags != VRING_EVENT_F_DESC) { |
| 1003 | if (vq->driver_event->flags != VRING_EVENT_F_DISABLE) |
| 1004 | kick = true; |
| 1005 | goto kick; |
| 1006 | } |
| 1007 | |
| 1008 | if (unlikely(!signalled_used_valid)) { |
| 1009 | kick = true; |
| 1010 | goto kick; |
| 1011 | } |
| 1012 | |
| 1013 | rte_atomic_thread_fence(rte_memory_order_acquire); |
| 1014 | |
| 1015 | off_wrap = vq->driver_event->off_wrap; |
| 1016 | off = off_wrap & ~(1 << 15); |
| 1017 | |
| 1018 | if (new <= old) |
| 1019 | old -= vq->size; |
| 1020 | |
| 1021 | if (vq->used_wrap_counter != off_wrap >> 15) |
| 1022 | off -= vq->size; |
| 1023 | |
| 1024 | if (vhost_need_event(off, new, old)) |
| 1025 | kick = true; |
| 1026 | kick: |
| 1027 | if (kick) |
| 1028 | vhost_vring_inject_irq(dev, vq); |
| 1029 | } |
| 1030 | |
| 1031 | static __rte_always_inline void |
| 1032 | free_ind_table(void *idesc) |
no test coverage detected