Function cbm_sem_random_index

src/semantic/semantic.c:437–466

}

void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) {
    memset(out, 0, sizeof(*out));
    if (!token) {
        return;
    }

    /* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */
    ensure_pretrained_map();
    const char *idx_str = cbm_ht_get(g_pretrained_map, token);
    if (idx_str) {
        char *end = NULL;
        long idx = strtol(idx_str, &end, BASE_DECIMAL);
        if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
            const int8_t *pvec = pretrained_vec_at((int)idx);
            for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) {
                out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX;
            }
            return;
        }
    }

    /* Fallback: sparse random vector for tokens not in pretrained vocab. */
    uint64_t seed = XXH3_64bits(token, strlen(token));
    for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
        uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
        int pos = (int)(h % CBM_SEM_DIM);
        float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
        out->v[pos] += sign;
    }
}

void cbm_sem_normalize(cbm_sem_vec_t *v) {
    if (!v) {