Function build_src_entry

src/semantic/semantic.c:1101–1155

Build one src_entry for a token: dense float32 reference if in nomic vocab, * sparse inline representation otherwise. Collisions in the sparse hash are * merged and zeros filtered so the final representation is exactly the same * mathematical vector that the old dense path produced. */

 * merged and zeros filtered so the final representation is exactly the same
 * mathematical vector that the old dense path produced. */
static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) {
    memset(out, 0, sizeof(*out));
    if (!token) {
        out->is_sparse = SKIP_ONE;
        out->nnz = 0;
        return;
    }
    /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */
    const char *idx_str = cbm_ht_get(g_pretrained_map, token);
    if (idx_str) {
        char *end = NULL;
        long idx = strtol(idx_str, &end, BASE_DECIMAL);
        if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
            out->is_sparse = 0;
            out->dense_int8 = pretrained_vec_at((int)idx);
            return;
        }
    }
    /* Sparse path: compute 8 hash positions with collision merging. */
    out->is_sparse = SKIP_ONE;
    uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE];
    float tmp_val[CBM_SEM_SPARSE_NNZE];
    int count = 0;
    uint64_t seed = XXH3_64bits(token, strlen(token));
    for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
        uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
        int pos = (int)(h % CBM_SEM_DIM);
        float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
        /* Merge collisions */
        int found = CBM_NOT_FOUND;
        for (int j = 0; j < count; j++) {
            if (tmp_idx[j] == (uint16_t)pos) {
                found = j;
                break;
            }
        }
        if (found >= 0) {
            tmp_val[found] += sign;
        } else {
            tmp_idx[count] = (uint16_t)pos;
            tmp_val[count] = sign;
            count++;
        }
    }
    /* Filter zeros */
    int nnz = 0;
    for (int j = 0; j < count; j++) {
        if (tmp_val[j] != 0.0F) {
            out->indices[nnz] = tmp_idx[j];
            out->values[nnz] = tmp_val[j];
            nnz++;
        }
    }
    out->nnz = (uint8_t)nnz;
}

static void src_build_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;