| 765 | } |
| 766 | |
| 767 | static void __noinline |
| 768 | __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v |
| 769 | LOCK_FILE_LINE_ARG_DEF) |
| 770 | { |
| 771 | struct turnstile *ts; |
| 772 | uintptr_t setv, queue; |
| 773 | |
| 774 | if (SCHEDULER_STOPPED()) |
| 775 | return; |
| 776 | |
| 777 | if (__rw_runlock_try(rw, td, &v)) |
| 778 | goto out_lockstat; |
| 779 | |
| 780 | /* |
| 781 | * Ok, we know we have waiters and we think we are the |
| 782 | * last reader, so grab the turnstile lock. |
| 783 | */ |
| 784 | turnstile_chain_lock(&rw->lock_object); |
| 785 | v = RW_READ_VALUE(rw); |
| 786 | for (;;) { |
| 787 | if (__rw_runlock_try(rw, td, &v)) |
| 788 | break; |
| 789 | |
| 790 | MPASS(v & RW_LOCK_WAITERS); |
| 791 | |
| 792 | /* |
| 793 | * Try to drop our lock leaving the lock in a unlocked |
| 794 | * state. |
| 795 | * |
| 796 | * If you wanted to do explicit lock handoff you'd have to |
| 797 | * do it here. You'd also want to use turnstile_signal() |
| 798 | * and you'd have to handle the race where a higher |
| 799 | * priority thread blocks on the write lock before the |
| 800 | * thread you wakeup actually runs and have the new thread |
| 801 | * "steal" the lock. For now it's a lot simpler to just |
| 802 | * wakeup all of the waiters. |
| 803 | * |
| 804 | * As above, if we fail, then another thread might have |
| 805 | * acquired a read lock, so drop the turnstile lock and |
| 806 | * restart. |
| 807 | */ |
| 808 | setv = RW_UNLOCKED; |
| 809 | queue = TS_SHARED_QUEUE; |
| 810 | if (v & RW_LOCK_WRITE_WAITERS) { |
| 811 | queue = TS_EXCLUSIVE_QUEUE; |
| 812 | setv |= (v & RW_LOCK_READ_WAITERS); |
| 813 | } |
| 814 | setv |= (v & RW_LOCK_WRITE_SPINNER); |
| 815 | if (!atomic_fcmpset_rel_ptr(&rw->rw_lock, &v, setv)) |
| 816 | continue; |
| 817 | if (LOCK_LOG_TEST(&rw->lock_object, 0)) |
| 818 | CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", |
| 819 | __func__, rw); |
| 820 | |
| 821 | /* |
| 822 | * Ok. The lock is released and all that's left is to |
| 823 | * wake up the waiters. Note that the lock might not be |
| 824 | * free anymore, but in that case the writers will just |
no test coverage detected