| 165 | } |
| 166 | |
| 167 | static inline bool io_uring_peek_batch_cqe_(struct io_uring *ring, |
| 168 | struct io_uring_cqe **cqes, |
| 169 | unsigned *count) |
| 170 | { |
| 171 | unsigned ready = io_uring_cq_ready(ring); |
| 172 | unsigned head; |
| 173 | unsigned mask; |
| 174 | unsigned last; |
| 175 | unsigned nr; |
| 176 | |
| 177 | if (!ready) |
| 178 | return false; |
| 179 | |
| 180 | head = *ring->cq.khead; |
| 181 | mask = ring->cq.ring_mask; |
| 182 | if (!(ring->flags & IORING_SETUP_CQE_MIXED)) { |
| 183 | unsigned shift = io_uring_cqe_shift(ring); |
| 184 | |
| 185 | if (ready < *count) |
| 186 | *count = ready; |
| 187 | last = head + *count; |
| 188 | for (;head != last; head++) |
| 189 | *(cqes++) = &ring->cq.cqes[(head & mask) << shift]; |
| 190 | |
| 191 | return true; |
| 192 | } |
| 193 | |
| 194 | /* |
| 195 | * For mixed CQE rings, CQEs take up one or two slots, and the kernel |
| 196 | * may post skip entries to pad out the ring at wrap time. Only return |
| 197 | * pointers to real CQEs, with *count denoting the number of CQEs. |
| 198 | */ |
| 199 | last = head + ready; |
| 200 | nr = 0; |
| 201 | while (head != last && nr < *count) { |
| 202 | struct io_uring_cqe *cqe = &ring->cq.cqes[head & mask]; |
| 203 | |
| 204 | if (cqe->flags & IORING_CQE_F_SKIP) { |
| 205 | /* |
| 206 | * A skip entry can only be consumed if it's at the |
| 207 | * current CQ head, stop the batch otherwise. It'll |
| 208 | * be at the head for the next peek. |
| 209 | */ |
| 210 | if (nr) |
| 211 | break; |
| 212 | io_uring_cq_advance(ring, 1); |
| 213 | head++; |
| 214 | continue; |
| 215 | } |
| 216 | head += io_uring_cqe_nr(cqe); |
| 217 | cqes[nr++] = cqe; |
| 218 | } |
| 219 | *count = nr; |
| 220 | return nr != 0; |
| 221 | } |
| 222 | |
| 223 | /* |
| 224 | * Fill in an array of IO completions up to count, if any are available. |
no test coverage detected