MCPcopy Create free account
hub / github.com/DeusData/codebase-memory-mcp / finalize_pass2

Function finalize_pass2

src/semantic/semantic.c:1338–1391  ·  view source on GitHub ↗

Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */

Source from the content-addressed store, hash-verified

1336
1337/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */
1338static void finalize_pass2(finalize_params_t *p) {
1339 int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
1340 (size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t));
1341 if (pass1_q) {
1342 pass1_quant_ctx_t qc = {
1343 .entries = p->corpus->entries,
1344 .pass1_q = pass1_q,
1345 .entry_count = p->corpus->entry_count,
1346 };
1347 atomic_init(&qc.next_idx, 0);
1348 cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts);
1349 }
1350
1351 cbm_sem_vec_t *pass1 =
1352 cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t));
1353 if (pass1) {
1354 for (int i = 0; i < p->corpus->entry_count; i++) {
1355 pass1[i] = p->corpus->entries[i].enriched_vec;
1356 }
1357 }
1358
1359 if (pass1_q) {
1360 cooccur_int8_ctx_t cc = {
1361 .entries = p->corpus->entries,
1362 .pass1_q = pass1_q,
1363 .doc_token_ids = p->corpus->doc_token_ids,
1364 .doc_token_counts = p->corpus->doc_token_counts,
1365 .rev = p->rev,
1366 .doc_count = p->corpus->doc_count,
1367 .entry_count = p->corpus->entry_count,
1368 .num_chunks = p->num_chunks,
1369 .chunk_size = p->chunk_size,
1370 .tile_size = p->tile_size,
1371 };
1372 atomic_init(&cc.next_chunk, 0);
1373 cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts);
1374 }
1375
1376 if (pass1) {
1377 blend_ctx_t bc = {
1378 .entries = p->corpus->entries,
1379 .pass1 = pass1,
1380 .entry_count = p->corpus->entry_count,
1381 };
1382 atomic_init(&bc.next_idx, 0);
1383 cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts);
1384 cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1);
1385 }
1386 cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q);
1387
1388 norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
1389 atomic_init(&nc.next_idx, 0);
1390 cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
1391}
1392
1393/* The per-document token id lists exist for the co-occurrence pass only;
1394 * released at the end of finalize so the vector phase runs without them

Callers 1

cbm_sem_corpus_finalizeFunction · 0.85

Calls 3

cbm_allocFunction · 0.85
cbm_parallel_forFunction · 0.85
cbm_freeFunction · 0.85

Tested by

no test coverage detected