| 493 | } |
| 494 | |
| 495 | static inline void |
| 496 | virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, |
| 497 | struct rte_mbuf *cookie, |
| 498 | int in_order) |
| 499 | { |
| 500 | struct virtqueue *vq = virtnet_txq_to_vq(txvq); |
| 501 | struct vring_packed_desc *dp; |
| 502 | struct vq_desc_extra *dxp; |
| 503 | uint16_t idx, id, flags; |
| 504 | int16_t head_size = vq->hw->vtnet_hdr_size; |
| 505 | struct virtio_net_hdr *hdr; |
| 506 | |
| 507 | id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; |
| 508 | idx = vq->vq_avail_idx; |
| 509 | dp = &vq->vq_packed.ring.desc[idx]; |
| 510 | |
| 511 | dxp = &vq->vq_descx[id]; |
| 512 | dxp->ndescs = 1; |
| 513 | dxp->cookie = cookie; |
| 514 | |
| 515 | flags = vq->vq_packed.cached_flags; |
| 516 | |
| 517 | /* prepend cannot fail, checked by caller */ |
| 518 | hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, |
| 519 | -head_size); |
| 520 | |
| 521 | /* if offload disabled, hdr is not zeroed yet, do it now */ |
| 522 | if (!vq->hw->has_tx_offload) |
| 523 | virtqueue_clear_net_hdr(hdr); |
| 524 | else |
| 525 | virtqueue_xmit_offload(hdr, cookie); |
| 526 | |
| 527 | dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; |
| 528 | dp->len = cookie->data_len + head_size; |
| 529 | dp->id = id; |
| 530 | |
| 531 | if (++vq->vq_avail_idx >= vq->vq_nentries) { |
| 532 | vq->vq_avail_idx -= vq->vq_nentries; |
| 533 | vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; |
| 534 | } |
| 535 | |
| 536 | vq->vq_free_cnt--; |
| 537 | |
| 538 | if (!in_order) { |
| 539 | vq->vq_desc_head_idx = dxp->next; |
| 540 | if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) |
| 541 | vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; |
| 542 | } |
| 543 | |
| 544 | virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); |
| 545 | } |
| 546 | |
| 547 | static inline void |
| 548 | virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, |
no test coverage detected