| 967 | } |
| 968 | |
| 969 | static __noinline int |
| 970 | lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, |
| 971 | const char *file, int line, struct lockmgr_wait *lwa) |
| 972 | { |
| 973 | uintptr_t tid, v, setv; |
| 974 | int error = 0; |
| 975 | int op; |
| 976 | |
| 977 | if (KERNEL_PANICKED()) |
| 978 | goto out; |
| 979 | |
| 980 | tid = (uintptr_t)curthread; |
| 981 | |
| 982 | _lockmgr_assert(lk, KA_SLOCKED, file, line); |
| 983 | |
| 984 | op = flags & LK_TYPE_MASK; |
| 985 | v = lockmgr_read_value(lk); |
| 986 | for (;;) { |
| 987 | if (LK_SHARERS(v) > 1) { |
| 988 | if (op == LK_TRYUPGRADE) { |
| 989 | LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", |
| 990 | __func__, lk); |
| 991 | error = EBUSY; |
| 992 | goto out; |
| 993 | } |
| 994 | if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &v, |
| 995 | v - LK_ONE_SHARER)) { |
| 996 | lockmgr_note_shared_release(lk, file, line); |
| 997 | goto out_xlock; |
| 998 | } |
| 999 | continue; |
| 1000 | } |
| 1001 | MPASS((v & ~LK_ALL_WAITERS) == LK_SHARERS_LOCK(1)); |
| 1002 | |
| 1003 | setv = tid; |
| 1004 | setv |= (v & LK_ALL_WAITERS); |
| 1005 | |
| 1006 | /* |
| 1007 | * Try to switch from one shared lock to an exclusive one. |
| 1008 | * We need to preserve waiters flags during the operation. |
| 1009 | */ |
| 1010 | if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) { |
| 1011 | LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, |
| 1012 | line); |
| 1013 | WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | |
| 1014 | LK_TRYWIT(flags), file, line); |
| 1015 | LOCKSTAT_RECORD0(lockmgr__upgrade, lk); |
| 1016 | TD_SLOCKS_DEC(curthread); |
| 1017 | goto out; |
| 1018 | } |
| 1019 | } |
| 1020 | |
| 1021 | out_xlock: |
| 1022 | error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); |
| 1023 | flags &= ~LK_INTERLOCK; |
| 1024 | out: |
| 1025 | lockmgr_exit(flags, ilk, 0); |
| 1026 | return (error); |
no test coverage detected