Function tokenize_worker

src/pipeline/pass_semantic_edges.c:611–641

} tokenize_ctx_t;

static void tokenize_worker(int worker_id, void *ctx_ptr) {
    tokenize_ctx_t *tc = ctx_ptr;
    while (true) {
        int f = atomic_fetch_add_explicit(&tc->next_idx, SKIP_ONE, memory_order_relaxed);
        if (f >= tc->func_count) {
            break;
        }

        const cbm_gbuf_node_t *n = tc->node_ptrs[f];
        /* Write directly into the shared buffer slice for this function — the
         * strdup'd tokens are owned by all_tokens[] from the moment they land
         * in this slot, which avoids a spurious analyzer "leak" diagnostic on
         * the previous stack-local relay pattern. */
        char **dst = &tc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS];
        int count = tokenize_node(n, tc->gbuf, dst, CBM_SEM_MAX_TOKENS);
        count = inject_pattern_tokens(n, tc->gbuf, dst, count, CBM_SEM_MAX_TOKENS);
        if (tc->pools && tc->pools[worker_id]) {
            CBMHashTable *pool = tc->pools[worker_id];
            for (int t = 0; t < count; t++) {
                char *canon = cbm_ht_get(pool, dst[t]);
                if (canon) {
                    free(dst[t]);
                    dst[t] = canon;
                } else {
                    cbm_ht_set(pool, dst[t], dst[t]); /* key borrows the value */
                }
            }
        }
        tc->token_counts[f] = count;
    }
}

/* ── Parallel Phase 4: Build per-function vectors ────────────────── */