| 99 | } |
| 100 | |
| 101 | static __rte_always_inline int64_t |
| 102 | vhost_async_dma_transfer_one(struct virtio_net *dev, struct vhost_virtqueue *vq, |
| 103 | int16_t dma_id, uint16_t vchan_id, uint16_t flag_idx, |
| 104 | struct vhost_iov_iter *pkt) |
| 105 | __rte_shared_locks_required(&vq->access_lock) |
| 106 | { |
| 107 | struct async_dma_vchan_info *dma_info = &dma_copy_track[dma_id].vchans[vchan_id]; |
| 108 | uint16_t ring_mask = dma_info->ring_mask; |
| 109 | static bool vhost_async_dma_copy_log; |
| 110 | |
| 111 | |
| 112 | struct vhost_iovec *iov = pkt->iov; |
| 113 | int copy_idx = 0; |
| 114 | uint32_t nr_segs = pkt->nr_segs; |
| 115 | uint16_t i; |
| 116 | |
| 117 | if (rte_dma_burst_capacity(dma_id, vchan_id) < nr_segs) |
| 118 | return -1; |
| 119 | |
| 120 | for (i = 0; i < nr_segs; i++) { |
| 121 | copy_idx = rte_dma_copy(dma_id, vchan_id, (rte_iova_t)iov[i].src_addr, |
| 122 | (rte_iova_t)iov[i].dst_addr, iov[i].len, RTE_DMA_OP_FLAG_LLC); |
| 123 | /** |
| 124 | * Since all memory is pinned and DMA vChannel |
| 125 | * ring has enough space, failure should be a |
| 126 | * rare case. If failure happens, it means DMA |
| 127 | * device encounters serious errors; in this |
| 128 | * case, please stop async data-path and check |
| 129 | * what has happened to DMA device. |
| 130 | */ |
| 131 | if (unlikely(copy_idx < 0)) { |
| 132 | if (!vhost_async_dma_copy_log) { |
| 133 | VHOST_LOG_DATA(dev->ifname, ERR, |
| 134 | "DMA copy failed for channel %d:%u\n", |
| 135 | dma_id, vchan_id); |
| 136 | vhost_async_dma_copy_log = true; |
| 137 | } |
| 138 | return -1; |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | /** |
| 143 | * Only store packet completion flag address in the last copy's |
| 144 | * slot, and other slots are set to NULL. |
| 145 | */ |
| 146 | dma_info->pkts_cmpl_flag_addr[copy_idx & ring_mask] = &vq->async->pkts_cmpl_flag[flag_idx]; |
| 147 | |
| 148 | return nr_segs; |
| 149 | } |
| 150 | |
| 151 | static __rte_always_inline uint16_t |
| 152 | vhost_async_dma_transfer(struct virtio_net *dev, struct vhost_virtqueue *vq, |
no test coverage detected