| 1171 | } |
| 1172 | |
| 1173 | static __rte_always_inline void |
| 1174 | sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 1175 | struct rte_mbuf *m, uint32_t mbuf_offset, |
| 1176 | uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc) |
| 1177 | __rte_shared_locks_required(&vq->iotlb_lock) |
| 1178 | { |
| 1179 | struct batch_copy_elem *batch_copy = vq->batch_copy_elems; |
| 1180 | |
| 1181 | if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { |
| 1182 | if (to_desc) { |
| 1183 | rte_memcpy((void *)((uintptr_t)(buf_addr)), |
| 1184 | rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), |
| 1185 | cpy_len); |
| 1186 | vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len); |
| 1187 | PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0); |
| 1188 | } else { |
| 1189 | rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), |
| 1190 | (void *)((uintptr_t)(buf_addr)), |
| 1191 | cpy_len); |
| 1192 | } |
| 1193 | } else { |
| 1194 | if (to_desc) { |
| 1195 | batch_copy[vq->batch_copy_nb_elems].dst = |
| 1196 | (void *)((uintptr_t)(buf_addr)); |
| 1197 | batch_copy[vq->batch_copy_nb_elems].src = |
| 1198 | rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); |
| 1199 | batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; |
| 1200 | } else { |
| 1201 | batch_copy[vq->batch_copy_nb_elems].dst = |
| 1202 | rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); |
| 1203 | batch_copy[vq->batch_copy_nb_elems].src = |
| 1204 | (void *)((uintptr_t)(buf_addr)); |
| 1205 | } |
| 1206 | batch_copy[vq->batch_copy_nb_elems].len = cpy_len; |
| 1207 | vq->batch_copy_nb_elems++; |
| 1208 | } |
| 1209 | } |
| 1210 | |
| 1211 | static __rte_always_inline int |
| 1212 | mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, |
no test coverage detected