MCPcopy Create free account
hub / github.com/DeusData/codebase-memory-mcp / build_reverse_index

Function build_reverse_index

src/semantic/semantic.c:1218–1281  ·  view source on GitHub ↗

Build reverse index: token_id → list of (doc_id, position) pairs. * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */

Source from the content-addressed store, hash-verified

1216/* Build reverse index: token_id → list of (doc_id, position) pairs.
1217 * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */
1218static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
1219 reverse_index_t *rev = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(reverse_index_t));
1220 if (!rev) {
1221 return NULL;
1222 }
1223 /* Phase A: count occurrences per token */
1224 int *counts =
1225 cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
1226 if (!counts) {
1227 cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
1228 return NULL;
1229 }
1230 long total = 0;
1231 for (int d = 0; d < corpus->doc_count; d++) {
1232 int *ids = corpus->doc_token_ids[d];
1233 int len = corpus->doc_token_counts[d];
1234 for (int i = 0; i < len; i++) {
1235 int tid = ids[i];
1236 if (tid >= 0 && tid < corpus->entry_count) {
1237 counts[tid]++;
1238 total++;
1239 }
1240 }
1241 }
1242 /* Phase B: exclusive prefix sum → offsets[] */
1243 rev->offsets =
1244 cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
1245 if (!rev->offsets) {
1246 cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
1247 cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
1248 return NULL;
1249 }
1250 int running = 0;
1251 for (int t = 0; t < corpus->entry_count; t++) {
1252 rev->offsets[t] = running;
1253 running += counts[t];
1254 counts[t] = 0; /* reuse as per-token fill cursor */
1255 }
1256 rev->offsets[corpus->entry_count] = running;
1257 /* Phase C: fill flat array. Ensure allocation size > 0 even for empty
1258 * corpora (avoids cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 0) which is implementation-defined). */
1259 size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t);
1260 rev->flat = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, flat_bytes);
1261 if (!rev->flat) {
1262 cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
1263 cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
1264 cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
1265 return NULL;
1266 }
1267 for (int d = 0; d < corpus->doc_count; d++) {
1268 int *ids = corpus->doc_token_ids[d];
1269 int len = corpus->doc_token_counts[d];
1270 for (int i = 0; i < len; i++) {
1271 int tid = ids[i];
1272 if (tid >= 0 && tid < corpus->entry_count) {
1273 int slot = rev->offsets[tid] + counts[tid]++;
1274 rev->flat[slot].doc_id = (int32_t)d;
1275 rev->flat[slot].pos = (int32_t)i;

Callers 1

cbm_sem_corpus_finalizeFunction · 0.85

Calls 3

cbm_callocFunction · 0.85
cbm_freeFunction · 0.85
cbm_allocFunction · 0.85

Tested by

no test coverage detected