| 41 | |
| 42 | __use_avx2 |
| 43 | static __rte_always_inline void |
| 44 | __submit(struct idxd_dmadev *idxd) |
| 45 | { |
| 46 | rte_prefetch1(&idxd->batch_comp_ring[idxd->batch_idx_read]); |
| 47 | |
| 48 | if (idxd->batch_size == 0) |
| 49 | return; |
| 50 | |
| 51 | /* write completion to batch comp ring */ |
| 52 | rte_iova_t comp_addr = idxd->batch_iova + |
| 53 | (idxd->batch_idx_write * sizeof(struct idxd_completion)); |
| 54 | |
| 55 | if (idxd->batch_size == 1) { |
| 56 | /* submit batch directly */ |
| 57 | struct idxd_hw_desc desc = |
| 58 | idxd->desc_ring[idxd->batch_start & idxd->desc_ring_mask]; |
| 59 | desc.completion = comp_addr; |
| 60 | desc.op_flags |= IDXD_FLAG_REQUEST_COMPLETION; |
| 61 | _mm_sfence(); /* fence before writing desc to device */ |
| 62 | __idxd_movdir64b(idxd->portal, &desc); |
| 63 | } else { |
| 64 | const struct idxd_hw_desc batch_desc = { |
| 65 | .op_flags = (idxd_op_batch << IDXD_CMD_OP_SHIFT) | |
| 66 | IDXD_FLAG_COMPLETION_ADDR_VALID | |
| 67 | IDXD_FLAG_REQUEST_COMPLETION, |
| 68 | .desc_addr = __desc_idx_to_iova(idxd, |
| 69 | idxd->batch_start & idxd->desc_ring_mask), |
| 70 | .completion = comp_addr, |
| 71 | .size = idxd->batch_size, |
| 72 | }; |
| 73 | _mm_sfence(); /* fence before writing desc to device */ |
| 74 | __idxd_movdir64b(idxd->portal, &batch_desc); |
| 75 | } |
| 76 | |
| 77 | if (++idxd->batch_idx_write > idxd->max_batches) |
| 78 | idxd->batch_idx_write = 0; |
| 79 | |
| 80 | idxd->stats.submitted += idxd->batch_size; |
| 81 | |
| 82 | idxd->batch_start += idxd->batch_size; |
| 83 | idxd->batch_size = 0; |
| 84 | idxd->batch_idx_ring[idxd->batch_idx_write] = idxd->batch_start; |
| 85 | _mm256_store_si256((void *)&idxd->batch_comp_ring[idxd->batch_idx_write], |
| 86 | _mm256_setzero_si256()); |
| 87 | } |
| 88 | |
| 89 | __use_avx2 |
| 90 | static __rte_always_inline int |
no test coverage detected