| 1129 | } |
| 1130 | |
| 1131 | static __rte_always_inline int |
| 1132 | async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 1133 | struct rte_mbuf *m, uint32_t mbuf_offset, |
| 1134 | uint64_t buf_iova, uint32_t cpy_len, bool to_desc) |
| 1135 | __rte_shared_locks_required(&vq->access_lock) |
| 1136 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 1137 | { |
| 1138 | struct vhost_async *async = vq->async; |
| 1139 | uint64_t mapped_len; |
| 1140 | uint32_t buf_offset = 0; |
| 1141 | void *src, *dst; |
| 1142 | void *host_iova; |
| 1143 | |
| 1144 | while (cpy_len) { |
| 1145 | host_iova = (void *)(uintptr_t)gpa_to_first_hpa(dev, |
| 1146 | buf_iova + buf_offset, cpy_len, &mapped_len); |
| 1147 | if (unlikely(!host_iova)) { |
| 1148 | VHOST_LOG_DATA(dev->ifname, ERR, |
| 1149 | "%s: failed to get host iova.\n", |
| 1150 | __func__); |
| 1151 | return -1; |
| 1152 | } |
| 1153 | |
| 1154 | if (to_desc) { |
| 1155 | src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset); |
| 1156 | dst = host_iova; |
| 1157 | } else { |
| 1158 | src = host_iova; |
| 1159 | dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset); |
| 1160 | } |
| 1161 | |
| 1162 | if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len))) |
| 1163 | return -1; |
| 1164 | |
| 1165 | cpy_len -= (uint32_t)mapped_len; |
| 1166 | mbuf_offset += (uint32_t)mapped_len; |
| 1167 | buf_offset += (uint32_t)mapped_len; |
| 1168 | } |
| 1169 | |
| 1170 | return 0; |
| 1171 | } |
| 1172 | |
| 1173 | static __rte_always_inline void |
| 1174 | sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, |
no test coverage detected