| 88 | |
| 89 | __use_avx2 |
| 90 | static __rte_always_inline int |
| 91 | __idxd_write_desc(struct idxd_dmadev *idxd, |
| 92 | const uint32_t op_flags, |
| 93 | const rte_iova_t src, |
| 94 | const rte_iova_t dst, |
| 95 | const uint32_t size, |
| 96 | const uint32_t flags) |
| 97 | { |
| 98 | uint16_t mask = idxd->desc_ring_mask; |
| 99 | uint16_t job_id = idxd->batch_start + idxd->batch_size; |
| 100 | /* we never wrap batches, so we only mask the start and allow start+size to overflow */ |
| 101 | uint16_t write_idx = (idxd->batch_start & mask) + idxd->batch_size; |
| 102 | |
| 103 | /* first check batch ring space then desc ring space */ |
| 104 | if ((idxd->batch_idx_read == 0 && idxd->batch_idx_write == idxd->max_batches) || |
| 105 | idxd->batch_idx_write + 1 == idxd->batch_idx_read) |
| 106 | return -ENOSPC; |
| 107 | if (((write_idx + 1) & mask) == (idxd->ids_returned & mask)) |
| 108 | return -ENOSPC; |
| 109 | |
| 110 | /* write desc. Note: descriptors don't wrap, but the completion address does */ |
| 111 | const uint64_t op_flags64 = (uint64_t)(op_flags | IDXD_FLAG_COMPLETION_ADDR_VALID) << 32; |
| 112 | const uint64_t comp_addr = __desc_idx_to_iova(idxd, write_idx & mask); |
| 113 | _mm256_store_si256((void *)&idxd->desc_ring[write_idx], |
| 114 | _mm256_set_epi64x(dst, src, comp_addr, op_flags64)); |
| 115 | _mm256_store_si256((void *)&idxd->desc_ring[write_idx].size, |
| 116 | _mm256_set_epi64x(0, 0, 0, size)); |
| 117 | |
| 118 | idxd->batch_size++; |
| 119 | |
| 120 | rte_prefetch0_write(&idxd->desc_ring[write_idx + 1]); |
| 121 | |
| 122 | if (flags & RTE_DMA_OP_FLAG_SUBMIT) |
| 123 | __submit(idxd); |
| 124 | |
| 125 | return job_id; |
| 126 | } |
| 127 | |
| 128 | __use_avx2 |
| 129 | int |
no test coverage detected