Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */
| 1336 | |
| 1337 | /* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ |
| 1338 | static void finalize_pass2(finalize_params_t *p) { |
| 1339 | int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, |
| 1340 | (size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); |
| 1341 | if (pass1_q) { |
| 1342 | pass1_quant_ctx_t qc = { |
| 1343 | .entries = p->corpus->entries, |
| 1344 | .pass1_q = pass1_q, |
| 1345 | .entry_count = p->corpus->entry_count, |
| 1346 | }; |
| 1347 | atomic_init(&qc.next_idx, 0); |
| 1348 | cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); |
| 1349 | } |
| 1350 | |
| 1351 | cbm_sem_vec_t *pass1 = |
| 1352 | cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); |
| 1353 | if (pass1) { |
| 1354 | for (int i = 0; i < p->corpus->entry_count; i++) { |
| 1355 | pass1[i] = p->corpus->entries[i].enriched_vec; |
| 1356 | } |
| 1357 | } |
| 1358 | |
| 1359 | if (pass1_q) { |
| 1360 | cooccur_int8_ctx_t cc = { |
| 1361 | .entries = p->corpus->entries, |
| 1362 | .pass1_q = pass1_q, |
| 1363 | .doc_token_ids = p->corpus->doc_token_ids, |
| 1364 | .doc_token_counts = p->corpus->doc_token_counts, |
| 1365 | .rev = p->rev, |
| 1366 | .doc_count = p->corpus->doc_count, |
| 1367 | .entry_count = p->corpus->entry_count, |
| 1368 | .num_chunks = p->num_chunks, |
| 1369 | .chunk_size = p->chunk_size, |
| 1370 | .tile_size = p->tile_size, |
| 1371 | }; |
| 1372 | atomic_init(&cc.next_chunk, 0); |
| 1373 | cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts); |
| 1374 | } |
| 1375 | |
| 1376 | if (pass1) { |
| 1377 | blend_ctx_t bc = { |
| 1378 | .entries = p->corpus->entries, |
| 1379 | .pass1 = pass1, |
| 1380 | .entry_count = p->corpus->entry_count, |
| 1381 | }; |
| 1382 | atomic_init(&bc.next_idx, 0); |
| 1383 | cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); |
| 1384 | cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1); |
| 1385 | } |
| 1386 | cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q); |
| 1387 | |
| 1388 | norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; |
| 1389 | atomic_init(&nc.next_idx, 0); |
| 1390 | cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); |
| 1391 | } |
| 1392 | |
| 1393 | /* The per-document token id lists exist for the co-occurrence pass only; |
| 1394 | * released at the end of finalize so the vector phase runs without them |
no test coverage detected