* Wakeup all threads on the pending list and adjust the priority of the * current thread appropriately. This must be called with the turnstile * chain locked. */
| 945 | * chain locked. |
| 946 | */ |
| 947 | void |
| 948 | turnstile_unpend(struct turnstile *ts) |
| 949 | { |
| 950 | TAILQ_HEAD( ,thread) pending_threads; |
| 951 | struct thread *td; |
| 952 | u_char pri; |
| 953 | |
| 954 | MPASS(ts != NULL); |
| 955 | mtx_assert(&ts->ts_lock, MA_OWNED); |
| 956 | MPASS(ts->ts_owner == curthread || ts->ts_owner == NULL); |
| 957 | MPASS(!TAILQ_EMPTY(&ts->ts_pending)); |
| 958 | |
| 959 | /* |
| 960 | * Move the list of pending threads out of the turnstile and |
| 961 | * into a local variable. |
| 962 | */ |
| 963 | TAILQ_INIT(&pending_threads); |
| 964 | TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq); |
| 965 | #ifdef INVARIANTS |
| 966 | if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) && |
| 967 | TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE])) |
| 968 | ts->ts_lockobj = NULL; |
| 969 | #endif |
| 970 | /* |
| 971 | * Adjust the priority of curthread based on other contested |
| 972 | * locks it owns. Don't lower the priority below the base |
| 973 | * priority however. |
| 974 | */ |
| 975 | td = curthread; |
| 976 | thread_lock(td); |
| 977 | mtx_lock_spin(&td_contested_lock); |
| 978 | /* |
| 979 | * Remove the turnstile from this thread's list of contested locks |
| 980 | * since this thread doesn't own it anymore. New threads will |
| 981 | * not be blocking on the turnstile until it is claimed by a new |
| 982 | * owner. There might not be a current owner if this is a shared |
| 983 | * lock. |
| 984 | */ |
| 985 | if (ts->ts_owner != NULL) { |
| 986 | ts->ts_owner = NULL; |
| 987 | LIST_REMOVE(ts, ts_link); |
| 988 | } |
| 989 | pri = turnstile_calc_unlend_prio_locked(td); |
| 990 | mtx_unlock_spin(&td_contested_lock); |
| 991 | sched_unlend_prio(td, pri); |
| 992 | thread_unlock(td); |
| 993 | /* |
| 994 | * Wake up all the pending threads. If a thread is not blocked |
| 995 | * on a lock, then it is currently executing on another CPU in |
| 996 | * turnstile_wait() or sitting on a run queue waiting to resume |
| 997 | * in turnstile_wait(). Set a flag to force it to try to acquire |
| 998 | * the lock again instead of blocking. |
| 999 | */ |
| 1000 | while (!TAILQ_EMPTY(&pending_threads)) { |
| 1001 | td = TAILQ_FIRST(&pending_threads); |
| 1002 | TAILQ_REMOVE(&pending_threads, td, td_lockq); |
| 1003 | SDT_PROBE2(sched, , , wakeup, td, td->td_proc); |
| 1004 | thread_lock_block_wait(td); |
no test coverage detected