| 1622 | } |
| 1623 | |
| 1624 | static int __handle_send(struct io_uring *ring, struct conn *c, |
| 1625 | struct conn_dir *cd, struct io_uring_cqe *cqe) |
| 1626 | { |
| 1627 | struct conn_dir *ocd; |
| 1628 | int bid, nr_packets; |
| 1629 | |
| 1630 | if (send_ring) { |
| 1631 | if (!(cqe->flags & IORING_CQE_F_BUFFER)) { |
| 1632 | fprintf(stderr, "no buffer in send?! %d\n", cqe->res); |
| 1633 | return 1; |
| 1634 | } |
| 1635 | bid = cqe->flags >> IORING_CQE_BUFFER_SHIFT; |
| 1636 | } else { |
| 1637 | bid = cqe_to_bid(cqe); |
| 1638 | } |
| 1639 | |
| 1640 | /* |
| 1641 | * CQE notifications only happen with send/sendmsg zerocopy. They |
| 1642 | * tell us that the data has been acked, and that hence the buffer |
| 1643 | * is now free to reuse. Waiting on an ACK for each packet will slow |
| 1644 | * us down tremendously, so do all of our sends and then wait for |
| 1645 | * the ACKs to come in. They tend to come in bundles anyway. Once |
| 1646 | * all acks are done (cd->snd_notif == 0), then fire off the next |
| 1647 | * receive. |
| 1648 | */ |
| 1649 | if (cqe->flags & IORING_CQE_F_NOTIF) { |
| 1650 | cd->snd_notif--; |
| 1651 | } else { |
| 1652 | if (cqe->res && cqe->res < buf_size) |
| 1653 | cd->snd_shrt++; |
| 1654 | |
| 1655 | /* |
| 1656 | * BIDI will use the same buffer pool and do sends on both CDs, |
| 1657 | * so can't reliably check. TODO. |
| 1658 | */ |
| 1659 | if (!bidi && send_ring && cd->snd_next_bid != -1 && |
| 1660 | bid != cd->snd_next_bid) { |
| 1661 | fprintf(stderr, "send bid %d, wanted %d at %lu\n", bid, |
| 1662 | cd->snd_next_bid, cd->out_bytes); |
| 1663 | goto out_close; |
| 1664 | } |
| 1665 | |
| 1666 | assert(bid <= nr_bufs); |
| 1667 | |
| 1668 | vlog("send: got %d, %lu\n", cqe->res, cd->out_bytes); |
| 1669 | |
| 1670 | if (buf_ring_inc) |
| 1671 | nr_packets = handle_send_inc(c, cd, bid, cqe); |
| 1672 | else if (send_ring) |
| 1673 | nr_packets = handle_send_ring(c, cd, bid, cqe->res); |
| 1674 | else |
| 1675 | nr_packets = handle_send_buf(c, cd, bid, cqe->res); |
| 1676 | |
| 1677 | if (cd->snd_bucket) |
| 1678 | cd->snd_bucket[nr_packets]++; |
| 1679 | |
| 1680 | cd->out_buffers -= nr_packets; |
| 1681 | assert(cd->out_buffers >= 0); |
no test coverage detected