| 1127 | } |
| 1128 | |
| 1129 | static void extract_worker(int worker_id, void *ctx_ptr) { |
| 1130 | extract_ctx_t *ec = ctx_ptr; |
| 1131 | extract_worker_state_t *ws = &ec->workers[worker_id]; |
| 1132 | |
| 1133 | /* Lazy gbuf creation */ |
| 1134 | if (!ws->local_gbuf) { |
| 1135 | ws->local_gbuf = cbm_gbuf_new_worker(ec->project_name, ec->repo_path, ec->shared_ids); |
| 1136 | } |
| 1137 | |
| 1138 | /* Pull files from shared atomic counter */ |
| 1139 | while (SKIP_ONE) { |
| 1140 | int sort_pos = |
| 1141 | atomic_fetch_add_explicit(&ec->next_file_idx, SKIP_ONE, memory_order_relaxed); |
| 1142 | if (sort_pos >= ec->file_count) { |
| 1143 | break; |
| 1144 | } |
| 1145 | cbm_scale_tick(&ec->scale, sort_pos); |
| 1146 | if (atomic_load_explicit(ec->cancelled, memory_order_relaxed) || |
| 1147 | atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) { |
| 1148 | break; |
| 1149 | } |
| 1150 | |
| 1151 | /* Memory back-pressure (large repos): if the process is over its RSS |
| 1152 | * budget, reclaim this thread's freed pages and nap so peer workers can |
| 1153 | * finish their current file and return memory before this worker adds |
| 1154 | * another parse working set. Caps the concurrent extraction transient |
| 1155 | * near the budget instead of letting all workers parse their biggest |
| 1156 | * files at once. Self-disabling when the budget is unset (tests) or RSS |
| 1157 | * is under budget; bounded spins avoid deadlock when the resident graph |
| 1158 | * is itself near budget. |
| 1159 | * |
| 1160 | * Futility latch: when a FULL nap cycle ends still over budget, the |
| 1161 | * resident floor — not transients — holds the memory; napping again on |
| 1162 | * the next pull cannot reclaim it and only idles workers (linux kernel: |
| 1163 | * one full cycle per pull ≈ 390 s at 79% avg CPU). The worker that |
| 1164 | * latches bp_futile (0→1) runs ONE confirmation cycle while its peers |
| 1165 | * skip the nap. A confirmation that ends under budget re-arms the gate |
| 1166 | * (RSS drained after all); one that still ends over budget fails the |
| 1167 | * attempt whole (decision A, #1997 #832): every worker stops pulling, |
| 1168 | * extract returns CBM_PIPELINE_ABORT_OVER_BUDGET, nothing is published |
| 1169 | * and the previously serving index keeps answering. */ |
| 1170 | if (cbm_mem_budget() > 0) { |
| 1171 | bool over = cbm_mem_over_budget(); |
| 1172 | pp_charge_probe(ec, over); |
| 1173 | /* Anticipation: the gate sees the crossing per file pull, and the |
| 1174 | * workers' in-flight files carry the charge past the line before |
| 1175 | * the first sweep lands (kernel, 15 GB budget: high-water 15.65 |
| 1176 | * GB, 4% over, all of it set in that window). Spill mode is |
| 1177 | * entered PP_SPILL_EARLY_DIV-th short of the budget, so results |
| 1178 | * stop accumulating in memory while the headroom still covers the |
| 1179 | * in-flight work. The nap / futility path still keys on the |
| 1180 | * budget itself. */ |
| 1181 | if (!over && !pp_spill_active(ec)) { |
| 1182 | size_t budget = cbm_mem_budget(); |
| 1183 | if (cbm_mem_charged() > budget - budget / PP_SPILL_EARLY_DIV) { |
| 1184 | pp_spill_enter(ec, "near_budget"); |
| 1185 | } |
| 1186 | } |
nothing calls this directly
no test coverage detected