MCPcopy Create free account
hub / github.com/alibaba/zvec / load_query

Method load_query

tools/core/txt_input_reader.h:58–135  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

56class TxtInputReader {
57 public:
58 bool load_query(const std::string &query_file, const std::string &first_sep,
59 const std::string &second_sep,
60 std::vector<std::vector<T>> &features,
61 std::vector<SparseData<T>> &sparse_data,
62 std::vector<std::vector<uint64_t>> &taglists) {
63 std::fstream qf(query_file, std::ios::in);
64
65 if (!qf.is_open()) {
66 std::cerr << "open query file failed! [" << query_file << "]"
67 << std::endl;
68 return false;
69 }
70
71 bool ret;
72 std::string buffer;
73 while (getline(qf, buffer)) {
74 buffer.erase(buffer.find_last_not_of('\n') + 1);
75 if (buffer.empty()) {
76 continue;
77 }
78 std::vector<std::string> res;
79 ailego::StringHelper::Split(buffer, first_sep, &res);
80 if (res.empty()) {
81 continue;
82 }
83 std::string feature_str = res[0];
84 if (res.size() > 1) {
85 feature_str = res[1];
86 }
87 std::vector<T> feature;
88 size_t dimension = 0;
89 ret = load_from_string(feature_str, second_sep, feature, &dimension);
90 if (!ret) {
91 return false;
92 }
93
94 features.emplace_back(feature);
95
96 uint64_t key = atol(res[0].c_str());
97
98 // load sparse feature
99 uint32_t sparse_count = 0;
100 std::vector<uint32_t> sparse_indices;
101 std::vector<T> sparse_feature;
102
103 if (res.size() >= 3) {
104 ret = load_from_string_sparse(key, res[2], second_sep, sparse_indices,
105 sparse_feature, &sparse_count);
106 if (!ret) {
107 std::cerr << "load sparse failed for key: " << key << std::endl;
108 return false;
109 }
110 }
111
112 sparse_data.emplace_back(sparse_count, std::move(sparse_indices),
113 std::move(sparse_feature));
114
115 if (res.size() >= 4) {

Callers

nothing calls this directly

Calls 6

SplitFunction · 0.85
eraseMethod · 0.80
c_strMethod · 0.80
emptyMethod · 0.45
sizeMethod · 0.45
closeMethod · 0.45

Tested by

no test coverage detected