* Search the tree of cpu_groups for the lowest or highest loaded cpu * according to the match argument. This routine actually compares the * load on all paths through the tree and finds the least loaded cpu on * the least loaded path, which may differ from the least loaded cpu in * the system. This balances work among caches and buses. * * This inline is instantiated in three forms below u
| 664 | * also recursive to the depth of the tree. |
| 665 | */ |
| 666 | static __always_inline int |
| 667 | cpu_search(const struct cpu_group *cg, struct cpu_search *low, |
| 668 | struct cpu_search *high, const int match) |
| 669 | { |
| 670 | struct cpu_search lgroup; |
| 671 | struct cpu_search hgroup; |
| 672 | cpuset_t cpumask; |
| 673 | struct cpu_group *child; |
| 674 | struct tdq *tdq; |
| 675 | int cpu, i, hload, lload, load, total, rnd; |
| 676 | |
| 677 | total = 0; |
| 678 | cpumask = cg->cg_mask; |
| 679 | if (match & CPU_SEARCH_LOWEST) { |
| 680 | lload = INT_MAX; |
| 681 | lgroup = *low; |
| 682 | } |
| 683 | if (match & CPU_SEARCH_HIGHEST) { |
| 684 | hload = INT_MIN; |
| 685 | hgroup = *high; |
| 686 | } |
| 687 | |
| 688 | /* Iterate through the child CPU groups and then remaining CPUs. */ |
| 689 | for (i = cg->cg_children, cpu = mp_maxid; ; ) { |
| 690 | if (i == 0) { |
| 691 | #ifdef HAVE_INLINE_FFSL |
| 692 | cpu = CPU_FFS(&cpumask) - 1; |
| 693 | #else |
| 694 | while (cpu >= 0 && !CPU_ISSET(cpu, &cpumask)) |
| 695 | cpu--; |
| 696 | #endif |
| 697 | if (cpu < 0) |
| 698 | break; |
| 699 | child = NULL; |
| 700 | } else |
| 701 | child = &cg->cg_child[i - 1]; |
| 702 | |
| 703 | if (match & CPU_SEARCH_LOWEST) |
| 704 | lgroup.cs_cpu = -1; |
| 705 | if (match & CPU_SEARCH_HIGHEST) |
| 706 | hgroup.cs_cpu = -1; |
| 707 | if (child) { /* Handle child CPU group. */ |
| 708 | CPU_ANDNOT(&cpumask, &child->cg_mask); |
| 709 | switch (match) { |
| 710 | case CPU_SEARCH_LOWEST: |
| 711 | load = cpu_search_lowest(child, &lgroup); |
| 712 | break; |
| 713 | case CPU_SEARCH_HIGHEST: |
| 714 | load = cpu_search_highest(child, &hgroup); |
| 715 | break; |
| 716 | case CPU_SEARCH_BOTH: |
| 717 | load = cpu_search_both(child, &lgroup, &hgroup); |
| 718 | break; |
| 719 | } |
| 720 | } else { /* Handle child CPU. */ |
| 721 | CPU_CLR(cpu, &cpumask); |
| 722 | tdq = TDQ_CPU(cpu); |
| 723 | load = tdq->tdq_load * 256; |
no test coverage detected