Function batch_resolve_one_doc

src/semantic/semantic.c:626–667

Resolve one document: look up each token's global ID, fill the corpus * doc_token_ids[d], and bump the per-token doc_freq counter atomically. The * caller is responsible for ensuring `seen` has capacity for `count` ints * before calling (the worker grows its per-thread scratch buffer). */

 * caller is responsible for ensuring `seen` has capacity for `count` ints
 * before calling (the worker grows its per-thread scratch buffer). */
static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) {
    int count = bc->token_counts[doc_index];
    if (count <= 0) {
        bc->corpus->doc_token_ids[doc_index] = NULL;
        bc->corpus->doc_token_counts[doc_index] = 0;
        return;
    }
    int *ids = malloc((size_t)count * sizeof(int));
    bc->corpus->doc_token_ids[doc_index] = ids;
    bc->corpus->doc_token_counts[doc_index] = count;

    int seen_count = 0;
    char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens];
    for (int i = 0; i < count; i++) {
        const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]);
        int tid = CBM_NOT_FOUND;
        if (idx_str) {
            char *end = NULL;
            long parsed = strtol(idx_str, &end, BASE_DECIMAL);
            if (end != idx_str) {
                tid = (int)parsed;
            }
        }
        ids[i] = tid;
        if (tid < 0) {
            continue;
        }
        /* Unique-per-doc check for IDF */
        bool is_new = true;
        for (int j = 0; j < seen_count; j++) {
            if (seen[j] == tid) {
                is_new = false;
                break;
            }
        }
        if (is_new) {
            seen[seen_count++] = tid;
            atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC,
                                      memory_order_relaxed);
        }
    }
}

static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;