| 658 | } |
| 659 | |
| 660 | std::vector<int64_t> |
| 661 | HGraph::build_by_odescent(const DatasetPtr& data) { |
| 662 | std::vector<int64_t> failed_ids; |
| 663 | |
| 664 | auto total = data->GetNumElements(); |
| 665 | const auto* labels = data->GetIds(); |
| 666 | const auto* vectors = data->GetFloat32Vectors(); |
| 667 | const auto* extra_infos = data->GetExtraInfos(); |
| 668 | Vector<int64_t> valid_indices(allocator_); |
| 669 | UnorderedSet<LabelType> seen_labels(allocator_); |
| 670 | for (int64_t i = 0; i < total; ++i) { |
| 671 | auto label = labels[i]; |
| 672 | if (this->label_table_->CheckLabel(label) or seen_labels.find(label) != seen_labels.end()) { |
| 673 | failed_ids.emplace_back(label); |
| 674 | continue; |
| 675 | } |
| 676 | seen_labels.insert(label); |
| 677 | valid_indices.emplace_back(i); |
| 678 | } |
| 679 | auto inner_ids = this->get_unique_inner_ids(static_cast<InnerIdType>(valid_indices.size())); |
| 680 | auto current_count = total_count_.load(); |
| 681 | uint64_t new_ids_count = 0; |
| 682 | for (auto inner_id : inner_ids) { |
| 683 | if (inner_id >= current_count) { |
| 684 | ++new_ids_count; |
| 685 | } |
| 686 | } |
| 687 | this->resize(current_count + new_ids_count); |
| 688 | this->total_count_ += new_ids_count; |
| 689 | Vector<Vector<InnerIdType>> route_graph_ids(allocator_); |
| 690 | for (InnerIdType cur_size = 0; cur_size < valid_indices.size(); ++cur_size) { |
| 691 | auto i = valid_indices[cur_size]; |
| 692 | auto label = labels[i]; |
| 693 | InnerIdType inner_id = inner_ids.at(cur_size); |
| 694 | this->label_table_->Insert(inner_id, label); |
| 695 | this->basic_flatten_codes_->InsertVector(vectors + dim_ * i, inner_id); |
| 696 | if (use_reorder_) { |
| 697 | this->high_precise_codes_->InsertVector(vectors + dim_ * i, inner_id); |
| 698 | } |
| 699 | if (create_new_raw_vector_) { |
| 700 | this->raw_vector_->InsertVector(vectors + dim_ * i, inner_id); |
| 701 | } |
| 702 | auto level = this->get_random_level() - 1; |
| 703 | if (level >= 0) { |
| 704 | if (level >= static_cast<int>(route_graph_ids.size()) || route_graph_ids.empty()) { |
| 705 | for (auto k = static_cast<int>(route_graph_ids.size()); k <= level; ++k) { |
| 706 | route_graph_ids.emplace_back(allocator_); |
| 707 | } |
| 708 | entry_point_id_ = inner_id; |
| 709 | } |
| 710 | for (int j = 0; j <= level; ++j) { |
| 711 | route_graph_ids[j].emplace_back(inner_id); |
| 712 | } |
| 713 | } |
| 714 | } |
| 715 | auto build_data = (use_reorder_ and not build_by_base_) ? this->high_precise_codes_ |
| 716 | : this->basic_flatten_codes_; |
| 717 | { |
no test coverage detected