MCPcopy Create free account
hub / github.com/alibaba/zvec / BuildQuantizedIndex

Function BuildQuantizedIndex

tests/core/algorithm/flat/flat_builder_test.cc:390–427  ·  view source on GitHub ↗

Create a quantizer and build a flat index with quantized datapoints

Source from the content-addressed store, hash-verified

388
389//! Create a quantizer and build a flat index with quantized datapoints
390static void BuildQuantizedIndex(const std::string &metric_name,
391 const std::vector<std::vector<float>> &data,
392 size_t dim, const std::string &path,
393 std::shared_ptr<zvec::turbo::Quantizer> *out,
394 const char *quantizer_name = "Fp32Quantizer") {
395 IndexMeta raw_meta;
396 raw_meta.set_meta(IndexMeta::DataType::DT_FP32, dim);
397 raw_meta.set_metric(metric_name, 0, Params());
398
399 auto quantizer = IndexFactory::CreateQuantizer(quantizer_name);
400 ASSERT_NE(nullptr, quantizer);
401 ASSERT_EQ(0, quantizer->init(raw_meta, Params()));
402
403 IndexMeta meta = quantizer->meta();
404 size_t code_bytes = quantizer->quantized_datapoint_vector_length();
405 ASSERT_EQ(code_bytes, meta.element_size());
406 meta.set_quantizer(quantizer_name, 0, Params());
407 meta.set_major_order(IndexMeta::MO_ROW);
408
409 auto holder = std::make_shared<QuantizedHolder>(meta);
410 NumericalVector<uint8_t> vec(code_bytes);
411 for (size_t i = 0; i < data.size(); ++i) {
412 quantizer->quantize_data(data[i].data(), &vec[0]);
413 ASSERT_TRUE(holder->emplace(i, vec));
414 }
415
416 auto builder = IndexFactory::CreateBuilder("FlatBuilder");
417 auto dumper = IndexFactory::CreateDumper("FileDumper");
418 ASSERT_NE(nullptr, builder);
419 ASSERT_NE(nullptr, dumper);
420
421 Params params;
422 ASSERT_EQ(0, builder->init(meta, params, quantizer));
423 ASSERT_EQ(0, dumper->create(path));
424 ASSERT_EQ(0, IndexBuilder::TrainBuildAndDump(builder, holder, dumper));
425 ASSERT_EQ(0, dumper->close());
426 *out = quantizer;
427}
428
429static void LoadQuantizedIndex(
430 const std::string &path,

Callers 1

TEST_FFunction · 0.85

Calls 15

ParamsFunction · 0.85
set_major_orderMethod · 0.80
set_metaMethod · 0.45
set_metricMethod · 0.45
initMethod · 0.45
metaMethod · 0.45
element_sizeMethod · 0.45
set_quantizerMethod · 0.45
sizeMethod · 0.45
quantize_dataMethod · 0.45
dataMethod · 0.45

Tested by

no test coverage detected