* Sync internal state with kernel ring state on the SQ side. Returns the * number of pending items in the SQ ring, for the shared ring. */
| 245 | * number of pending items in the SQ ring, for the shared ring. |
| 246 | */ |
| 247 | static unsigned __io_uring_flush_sq(struct io_uring *ring) |
| 248 | { |
| 249 | struct io_uring_sq *sq = &ring->sq; |
| 250 | unsigned tail = sq->sqe_tail; |
| 251 | |
| 252 | /* |
| 253 | * With IORING_SETUP_SQ_REWIND, the kernel ignores the SQ head / tail |
| 254 | * and submits entries from the beginning of the queue. Continue using |
| 255 | * the indices as before but reset the tail on submission. With the |
| 256 | * head kept to be zero, io_uring_get_sqe() / etc. will work without |
| 257 | * any extra changes. |
| 258 | */ |
| 259 | if (ring->flags & IORING_SETUP_SQ_REWIND) { |
| 260 | sq->sqe_tail = 0; |
| 261 | return tail; |
| 262 | } |
| 263 | |
| 264 | if (sq->sqe_head != tail) { |
| 265 | sq->sqe_head = tail; |
| 266 | /* |
| 267 | * Ensure kernel sees the SQE updates before the tail update. |
| 268 | */ |
| 269 | if (!(ring->flags & IORING_SETUP_SQPOLL)) |
| 270 | *sq->ktail = tail; |
| 271 | else |
| 272 | io_uring_smp_store_release(sq->ktail, tail); |
| 273 | } |
| 274 | /* |
| 275 | * This load needs to be atomic, since sq->khead is written concurrently |
| 276 | * by the kernel, but it doesn't need to be load_acquire, since the |
| 277 | * kernel doesn't store to the submission queue; it advances khead just |
| 278 | * to indicate that it's finished reading the submission queue entries |
| 279 | * so they're available for us to write to. |
| 280 | */ |
| 281 | return tail - IO_URING_READ_ONCE(*sq->khead); |
| 282 | } |
| 283 | |
| 284 | /* |
| 285 | * If we have kernel support for IORING_ENTER_EXT_ARG, then we can use that |
no outgoing calls
no test coverage detected