MCPcopy Create free account
hub / github.com/alibaba/zvec / load_query

Method load_query

tools/core/bench_original.cc:408–469  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

406 }
407
408 bool load_query(const std::string &query_file, const std::string &first_sep,
409 const std::string &second_sep) {
410 TxtInputReader<T> reader;
411 vector<vector<T>> queries;
412 vector<SparseData<T>> sparse_data;
413 vector<vector<uint64_t>> taglists;
414
415 if (!reader.load_query(query_file, first_sep, second_sep, queries,
416 sparse_data, taglists)) {
417 cerr << "Load query error" << endl;
418 return false;
419 }
420
421 if (batch_count_ == 1) {
422 for (size_t i = 0; i < sparse_data.size(); ++i) {
423 vector<uint32_t> sparse_count;
424 sparse_count.push_back(sparse_data[i].count);
425
426 batch_sparse_counts_.push_back(sparse_count);
427 batch_sparse_indices_.push_back(sparse_data[i].indices);
428 batch_sparse_features_.push_back(sparse_data[i].features);
429 }
430 } else {
431 size_t num_batch = (queries.size() + batch_count_ - 1) / batch_count_;
432 size_t idx = 0;
433 for (size_t n = 0; n < num_batch; ++n) {
434 vector<uint32_t> batch_sparse_count;
435 vector<uint32_t> batch_sparse_indices;
436 vector<T> batch_sparse_feature;
437
438 for (size_t i = 0; i < batch_count_; ++i) {
439 batch_sparse_count.push_back(sparse_data[idx].count);
440
441 for (size_t k = 0; k < sparse_data[idx].indices.size(); ++k) {
442 batch_sparse_indices.push_back(sparse_data[idx].indices[k]);
443 }
444
445 for (size_t k = 0; k < sparse_data[idx].features.size(); ++k) {
446 batch_sparse_feature.push_back(sparse_data[idx].features[k]);
447 }
448
449 idx = (idx + 1) % queries.size();
450 }
451
452 batch_sparse_counts_.push_back(batch_sparse_count);
453 batch_sparse_indices_.push_back(batch_sparse_indices);
454 batch_sparse_features_.push_back(batch_sparse_feature);
455 }
456 }
457
458 if (typeid(T) == typeid(float)) {
459 qmeta_.set_data_type(IndexMeta::DataType::DT_FP32);
460 } else if (typeid(T) == typeid(int8_t)) {
461 qmeta_.set_data_type(IndexMeta::DataType::DT_INT8);
462 } else {
463 cerr << "unsupported type";
464 return false;
465 }

Callers

nothing calls this directly

Calls 4

push_backMethod · 0.80
load_queryMethod · 0.45
sizeMethod · 0.45
set_data_typeMethod · 0.45

Tested by

no test coverage detected