* Sync internal state with kernel ring state on the SQ side. Returns the * number of pending items in the SQ ring, for the shared ring. */
| 277 | * number of pending items in the SQ ring, for the shared ring. |
| 278 | */ |
| 279 | unsigned __io_uring_flush_sq(struct io_uring *ring) |
| 280 | { |
| 281 | struct io_uring_sq *sq = &ring->sq; |
| 282 | unsigned tail = sq->sqe_tail; |
| 283 | |
| 284 | if (sq->sqe_head != tail) { |
| 285 | sq->sqe_head = tail; |
| 286 | /* |
| 287 | * Ensure kernel sees the SQE updates before the tail update. |
| 288 | */ |
| 289 | if (!(ring->flags & IORING_SETUP_SQPOLL)) |
| 290 | *sq->ktail = tail; |
| 291 | else |
| 292 | io_uring_smp_store_release(sq->ktail, tail); |
| 293 | } |
| 294 | /* |
| 295 | * This load needs to be atomic, since sq->khead is written concurrently |
| 296 | * by the kernel, but it doesn't need to be load_acquire, since the |
| 297 | * kernel doesn't store to the submission queue; it advances khead just |
| 298 | * to indicate that it's finished reading the submission queue entries |
| 299 | * so they're available for us to write to. |
| 300 | */ |
| 301 | return tail - IO_URING_READ_ONCE(*sq->khead); |
| 302 | } |
| 303 | |
| 304 | /* |
| 305 | * Implementation of error(3), prints an error message and exits. |
no outgoing calls