* This tdq has idled. Try to steal a thread from another cpu and switch * to it. */
| 981 | * to it. |
| 982 | */ |
| 983 | static int |
| 984 | tdq_idled(struct tdq *tdq) |
| 985 | { |
| 986 | struct cpu_group *cg; |
| 987 | struct tdq *steal; |
| 988 | cpuset_t mask; |
| 989 | int cpu, switchcnt; |
| 990 | |
| 991 | if (smp_started == 0 || steal_idle == 0 || tdq->tdq_cg == NULL) |
| 992 | return (1); |
| 993 | CPU_FILL(&mask); |
| 994 | CPU_CLR(PCPU_GET(cpuid), &mask); |
| 995 | restart: |
| 996 | switchcnt = tdq->tdq_switchcnt + tdq->tdq_oldswitchcnt; |
| 997 | for (cg = tdq->tdq_cg; ; ) { |
| 998 | cpu = sched_highest(cg, mask, steal_thresh); |
| 999 | /* |
| 1000 | * We were assigned a thread but not preempted. Returning |
| 1001 | * 0 here will cause our caller to switch to it. |
| 1002 | */ |
| 1003 | if (tdq->tdq_load) |
| 1004 | return (0); |
| 1005 | if (cpu == -1) { |
| 1006 | cg = cg->cg_parent; |
| 1007 | if (cg == NULL) |
| 1008 | return (1); |
| 1009 | continue; |
| 1010 | } |
| 1011 | steal = TDQ_CPU(cpu); |
| 1012 | /* |
| 1013 | * The data returned by sched_highest() is stale and |
| 1014 | * the chosen CPU no longer has an eligible thread. |
| 1015 | * |
| 1016 | * Testing this ahead of tdq_lock_pair() only catches |
| 1017 | * this situation about 20% of the time on an 8 core |
| 1018 | * 16 thread Ryzen 7, but it still helps performance. |
| 1019 | */ |
| 1020 | if (steal->tdq_load < steal_thresh || |
| 1021 | steal->tdq_transferable == 0) |
| 1022 | goto restart; |
| 1023 | tdq_lock_pair(tdq, steal); |
| 1024 | /* |
| 1025 | * We were assigned a thread while waiting for the locks. |
| 1026 | * Switch to it now instead of stealing a thread. |
| 1027 | */ |
| 1028 | if (tdq->tdq_load) |
| 1029 | break; |
| 1030 | /* |
| 1031 | * The data returned by sched_highest() is stale and |
| 1032 | * the chosen CPU no longer has an eligible thread, or |
| 1033 | * we were preempted and the CPU loading info may be out |
| 1034 | * of date. The latter is rare. In either case restart |
| 1035 | * the search. |
| 1036 | */ |
| 1037 | if (steal->tdq_load < steal_thresh || |
| 1038 | steal->tdq_transferable == 0 || |
| 1039 | switchcnt != tdq->tdq_switchcnt + tdq->tdq_oldswitchcnt) { |
| 1040 | tdq_unlock_pair(tdq, steal); |
no test coverage detected