Create a quantizer and build a flat index with quantized datapoints
| 388 | |
| 389 | //! Create a quantizer and build a flat index with quantized datapoints |
| 390 | static void BuildQuantizedIndex(const std::string &metric_name, |
| 391 | const std::vector<std::vector<float>> &data, |
| 392 | size_t dim, const std::string &path, |
| 393 | std::shared_ptr<zvec::turbo::Quantizer> *out, |
| 394 | const char *quantizer_name = "Fp32Quantizer") { |
| 395 | IndexMeta raw_meta; |
| 396 | raw_meta.set_meta(IndexMeta::DataType::DT_FP32, dim); |
| 397 | raw_meta.set_metric(metric_name, 0, Params()); |
| 398 | |
| 399 | auto quantizer = IndexFactory::CreateQuantizer(quantizer_name); |
| 400 | ASSERT_NE(nullptr, quantizer); |
| 401 | ASSERT_EQ(0, quantizer->init(raw_meta, Params())); |
| 402 | |
| 403 | IndexMeta meta = quantizer->meta(); |
| 404 | size_t code_bytes = quantizer->quantized_datapoint_vector_length(); |
| 405 | ASSERT_EQ(code_bytes, meta.element_size()); |
| 406 | meta.set_quantizer(quantizer_name, 0, Params()); |
| 407 | meta.set_major_order(IndexMeta::MO_ROW); |
| 408 | |
| 409 | auto holder = std::make_shared<QuantizedHolder>(meta); |
| 410 | NumericalVector<uint8_t> vec(code_bytes); |
| 411 | for (size_t i = 0; i < data.size(); ++i) { |
| 412 | quantizer->quantize_data(data[i].data(), &vec[0]); |
| 413 | ASSERT_TRUE(holder->emplace(i, vec)); |
| 414 | } |
| 415 | |
| 416 | auto builder = IndexFactory::CreateBuilder("FlatBuilder"); |
| 417 | auto dumper = IndexFactory::CreateDumper("FileDumper"); |
| 418 | ASSERT_NE(nullptr, builder); |
| 419 | ASSERT_NE(nullptr, dumper); |
| 420 | |
| 421 | Params params; |
| 422 | ASSERT_EQ(0, builder->init(meta, params, quantizer)); |
| 423 | ASSERT_EQ(0, dumper->create(path)); |
| 424 | ASSERT_EQ(0, IndexBuilder::TrainBuildAndDump(builder, holder, dumper)); |
| 425 | ASSERT_EQ(0, dumper->close()); |
| 426 | *out = quantizer; |
| 427 | } |
| 428 | |
| 429 | static void LoadQuantizedIndex( |
| 430 | const std::string &path, |
no test coverage detected