| 959 | } |
| 960 | |
| 961 | static __rte_always_inline int |
| 962 | fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 963 | uint16_t avail_idx, uint16_t *desc_count, |
| 964 | struct buf_vector *buf_vec, uint16_t *vec_idx, |
| 965 | uint16_t *buf_id, uint32_t *len, uint8_t perm) |
| 966 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 967 | { |
| 968 | bool wrap_counter = vq->avail_wrap_counter; |
| 969 | struct vring_packed_desc *descs = vq->desc_packed; |
| 970 | uint16_t vec_id = *vec_idx; |
| 971 | uint64_t dlen; |
| 972 | |
| 973 | if (avail_idx < vq->last_avail_idx) |
| 974 | wrap_counter ^= 1; |
| 975 | |
| 976 | /* |
| 977 | * Perform a load-acquire barrier in desc_is_avail to |
| 978 | * enforce the ordering between desc flags and desc |
| 979 | * content. |
| 980 | */ |
| 981 | if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter))) |
| 982 | return -1; |
| 983 | |
| 984 | *desc_count = 0; |
| 985 | *len = 0; |
| 986 | |
| 987 | while (1) { |
| 988 | if (unlikely(vec_id >= BUF_VECTOR_MAX)) |
| 989 | return -1; |
| 990 | |
| 991 | if (unlikely(*desc_count >= vq->size)) |
| 992 | return -1; |
| 993 | |
| 994 | *desc_count += 1; |
| 995 | *buf_id = descs[avail_idx].id; |
| 996 | |
| 997 | if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) { |
| 998 | if (unlikely(fill_vec_buf_packed_indirect(dev, vq, |
| 999 | &descs[avail_idx], |
| 1000 | &vec_id, buf_vec, |
| 1001 | len, perm) < 0)) |
| 1002 | return -1; |
| 1003 | } else { |
| 1004 | dlen = descs[avail_idx].len; |
| 1005 | *len += dlen; |
| 1006 | |
| 1007 | if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id, |
| 1008 | descs[avail_idx].addr, |
| 1009 | dlen, |
| 1010 | perm))) |
| 1011 | return -1; |
| 1012 | } |
| 1013 | |
| 1014 | if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0) |
| 1015 | break; |
| 1016 | |
| 1017 | if (++avail_idx >= vq->size) { |
| 1018 | avail_idx -= vq->size; |
no test coverage detected