MCPcopy Create free account
hub / github.com/alibaba/zvec / load_query

Method load_query

tools/core/bench.cc:366–442  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

364 }
365
366 bool load_query(const std::string &query_file, const std::string &first_sep,
367 const std::string &second_sep) {
368 TxtInputReader<T> reader;
369 vector<vector<T>> queries;
370 vector<SparseData<T>> sparse_data;
371 vector<vector<uint64_t>> taglists;
372
373 if (!reader.load_query(query_file, first_sep, second_sep, queries,
374 sparse_data, taglists)) {
375 LOG_ERROR("Load query error");
376 return false;
377 }
378
379 linear_sparse_data_ = sparse_data;
380
381 if (batch_count_ == 1) {
382 for (size_t i = 0; i < sparse_data.size(); ++i) {
383 vector<uint32_t> sparse_count;
384 sparse_count.push_back(sparse_data[i].count);
385
386 batch_sparse_counts_.push_back(sparse_count);
387 batch_sparse_indices_.push_back(sparse_data[i].indices);
388 batch_sparse_features_.push_back(sparse_data[i].features);
389 }
390
391 for (size_t i = 0; i < taglists.size(); ++i) {
392 vector<vector<uint64_t>> new_taglists;
393 new_taglists.push_back(taglists[i]);
394
395 batch_taglists_.push_back(std::move(new_taglists));
396 }
397 } else {
398 size_t num_batch = (queries.size() + batch_count_ - 1) / batch_count_;
399 size_t idx = 0;
400 for (size_t n = 0; n < num_batch; ++n) {
401 vector<uint32_t> batch_sparse_count;
402 vector<uint32_t> batch_sparse_indices;
403 vector<T> batch_sparse_feature;
404 vector<vector<uint64_t>> batch_taglists;
405
406 for (size_t i = 0; i < batch_count_; ++i) {
407 batch_sparse_count.push_back(sparse_data[idx].count);
408
409 for (size_t k = 0; k < sparse_data[idx].indices.size(); ++k) {
410 batch_sparse_indices.push_back(sparse_data[idx].indices[k]);
411 }
412
413 for (size_t k = 0; k < sparse_data[idx].features.size(); ++k) {
414 batch_sparse_feature.push_back(sparse_data[idx].features[k]);
415 }
416
417 if (taglists.size() > idx) {
418 batch_taglists.push_back(taglists[idx]);
419 }
420
421 idx = (idx + 1) % queries.size();
422 }
423

Callers

nothing calls this directly

Calls 4

push_backMethod · 0.80
load_queryMethod · 0.45
sizeMethod · 0.45
set_data_typeMethod · 0.45

Tested by

no test coverage detected