| 1089 | } |
| 1090 | |
| 1091 | void |
| 1092 | invlop_handler(void) |
| 1093 | { |
| 1094 | struct pcpu *initiator_pc; |
| 1095 | pmap_t smp_tlb_pmap; |
| 1096 | vm_offset_t smp_tlb_addr1, smp_tlb_addr2; |
| 1097 | u_int initiator_cpu_id; |
| 1098 | enum invl_op_codes smp_tlb_op; |
| 1099 | uint32_t *scoreboard, smp_tlb_gen; |
| 1100 | |
| 1101 | scoreboard = invl_scoreboard_getcpu(PCPU_GET(cpuid)); |
| 1102 | for (;;) { |
| 1103 | for (initiator_cpu_id = 0; initiator_cpu_id <= mp_maxid; |
| 1104 | initiator_cpu_id++) { |
| 1105 | if (atomic_load_int(&scoreboard[initiator_cpu_id]) == 0) |
| 1106 | break; |
| 1107 | } |
| 1108 | if (initiator_cpu_id > mp_maxid) |
| 1109 | break; |
| 1110 | initiator_pc = cpuid_to_pcpu[initiator_cpu_id]; |
| 1111 | |
| 1112 | /* |
| 1113 | * This acquire fence and its corresponding release |
| 1114 | * fence in smp_targeted_tlb_shootdown() is between |
| 1115 | * reading zero scoreboard slot and accessing PCPU of |
| 1116 | * initiator for pc_smp_tlb values. |
| 1117 | */ |
| 1118 | atomic_thread_fence_acq(); |
| 1119 | smp_tlb_pmap = initiator_pc->pc_smp_tlb_pmap; |
| 1120 | smp_tlb_addr1 = initiator_pc->pc_smp_tlb_addr1; |
| 1121 | smp_tlb_addr2 = initiator_pc->pc_smp_tlb_addr2; |
| 1122 | smp_tlb_op = initiator_pc->pc_smp_tlb_op; |
| 1123 | smp_tlb_gen = initiator_pc->pc_smp_tlb_gen; |
| 1124 | |
| 1125 | /* |
| 1126 | * Ensure that we do not make our scoreboard |
| 1127 | * notification visible to the initiator until the |
| 1128 | * pc_smp_tlb values are read. The corresponding |
| 1129 | * fence is implicitly provided by the barrier in the |
| 1130 | * IPI send operation before the APIC ICR register |
| 1131 | * write. |
| 1132 | * |
| 1133 | * As an optimization, the request is acknowledged |
| 1134 | * before the actual invalidation is performed. It is |
| 1135 | * safe because target CPU cannot return to userspace |
| 1136 | * before handler finishes. Only NMI can preempt the |
| 1137 | * handler, but NMI would see the kernel handler frame |
| 1138 | * and not touch not-invalidated user page table. |
| 1139 | */ |
| 1140 | atomic_thread_fence_acq(); |
| 1141 | atomic_store_int(&scoreboard[initiator_cpu_id], smp_tlb_gen); |
| 1142 | |
| 1143 | invlop_handler_one_req(smp_tlb_op, smp_tlb_pmap, smp_tlb_addr1, |
| 1144 | smp_tlb_addr2); |
| 1145 | } |
| 1146 | } |
no test coverage detected