* Apply deferred queue state updates to a page. */
| 3528 | * Apply deferred queue state updates to a page. |
| 3529 | */ |
| 3530 | static inline void |
| 3531 | vm_pqbatch_process_page(struct vm_pagequeue *pq, vm_page_t m, uint8_t queue) |
| 3532 | { |
| 3533 | vm_page_astate_t new, old; |
| 3534 | |
| 3535 | CRITICAL_ASSERT(curthread); |
| 3536 | vm_pagequeue_assert_locked(pq); |
| 3537 | KASSERT(queue < PQ_COUNT, |
| 3538 | ("%s: invalid queue index %d", __func__, queue)); |
| 3539 | KASSERT(pq == _vm_page_pagequeue(m, queue), |
| 3540 | ("%s: page %p does not belong to queue %p", __func__, m, pq)); |
| 3541 | |
| 3542 | for (old = vm_page_astate_load(m);;) { |
| 3543 | if (__predict_false(old.queue != queue || |
| 3544 | (old.flags & PGA_QUEUE_OP_MASK) == 0)) { |
| 3545 | counter_u64_add(queue_nops, 1); |
| 3546 | break; |
| 3547 | } |
| 3548 | KASSERT(old.queue != PQ_NONE || |
| 3549 | (old.flags & PGA_QUEUE_STATE_MASK) == 0, |
| 3550 | ("%s: page %p has unexpected queue state", __func__, m)); |
| 3551 | |
| 3552 | new = old; |
| 3553 | if ((old.flags & PGA_DEQUEUE) != 0) { |
| 3554 | new.flags &= ~PGA_QUEUE_OP_MASK; |
| 3555 | new.queue = PQ_NONE; |
| 3556 | if (__predict_true(_vm_page_pqstate_commit_dequeue(pq, |
| 3557 | m, &old, new))) { |
| 3558 | counter_u64_add(queue_ops, 1); |
| 3559 | break; |
| 3560 | } |
| 3561 | } else { |
| 3562 | new.flags &= ~(PGA_REQUEUE | PGA_REQUEUE_HEAD); |
| 3563 | if (__predict_true(_vm_page_pqstate_commit_requeue(pq, |
| 3564 | m, &old, new))) { |
| 3565 | counter_u64_add(queue_ops, 1); |
| 3566 | break; |
| 3567 | } |
| 3568 | } |
| 3569 | } |
| 3570 | } |
| 3571 | |
| 3572 | static void |
| 3573 | vm_pqbatch_process(struct vm_pagequeue *pq, struct vm_batchqueue *bq, |
no test coverage detected