| 406 | } |
| 407 | |
| 408 | bool load_query(const std::string &query_file, const std::string &first_sep, |
| 409 | const std::string &second_sep) { |
| 410 | TxtInputReader<T> reader; |
| 411 | vector<vector<T>> queries; |
| 412 | vector<SparseData<T>> sparse_data; |
| 413 | vector<vector<uint64_t>> taglists; |
| 414 | |
| 415 | if (!reader.load_query(query_file, first_sep, second_sep, queries, |
| 416 | sparse_data, taglists)) { |
| 417 | cerr << "Load query error" << endl; |
| 418 | return false; |
| 419 | } |
| 420 | |
| 421 | if (batch_count_ == 1) { |
| 422 | for (size_t i = 0; i < sparse_data.size(); ++i) { |
| 423 | vector<uint32_t> sparse_count; |
| 424 | sparse_count.push_back(sparse_data[i].count); |
| 425 | |
| 426 | batch_sparse_counts_.push_back(sparse_count); |
| 427 | batch_sparse_indices_.push_back(sparse_data[i].indices); |
| 428 | batch_sparse_features_.push_back(sparse_data[i].features); |
| 429 | } |
| 430 | } else { |
| 431 | size_t num_batch = (queries.size() + batch_count_ - 1) / batch_count_; |
| 432 | size_t idx = 0; |
| 433 | for (size_t n = 0; n < num_batch; ++n) { |
| 434 | vector<uint32_t> batch_sparse_count; |
| 435 | vector<uint32_t> batch_sparse_indices; |
| 436 | vector<T> batch_sparse_feature; |
| 437 | |
| 438 | for (size_t i = 0; i < batch_count_; ++i) { |
| 439 | batch_sparse_count.push_back(sparse_data[idx].count); |
| 440 | |
| 441 | for (size_t k = 0; k < sparse_data[idx].indices.size(); ++k) { |
| 442 | batch_sparse_indices.push_back(sparse_data[idx].indices[k]); |
| 443 | } |
| 444 | |
| 445 | for (size_t k = 0; k < sparse_data[idx].features.size(); ++k) { |
| 446 | batch_sparse_feature.push_back(sparse_data[idx].features[k]); |
| 447 | } |
| 448 | |
| 449 | idx = (idx + 1) % queries.size(); |
| 450 | } |
| 451 | |
| 452 | batch_sparse_counts_.push_back(batch_sparse_count); |
| 453 | batch_sparse_indices_.push_back(batch_sparse_indices); |
| 454 | batch_sparse_features_.push_back(batch_sparse_feature); |
| 455 | } |
| 456 | } |
| 457 | |
| 458 | if (typeid(T) == typeid(float)) { |
| 459 | qmeta_.set_data_type(IndexMeta::DataType::DT_FP32); |
| 460 | } else if (typeid(T) == typeid(int8_t)) { |
| 461 | qmeta_.set_data_type(IndexMeta::DataType::DT_INT8); |
| 462 | } else { |
| 463 | cerr << "unsupported type"; |
| 464 | return false; |
| 465 | } |
nothing calls this directly
no test coverage detected