| 203 | } |
| 204 | |
| 205 | static bool LoadFromTextFile(aitheta2::IndexThreads::TaskGroup *group, |
| 206 | index::Collection *collection) { |
| 207 | std::ifstream file_stream(FLAGS_file); |
| 208 | if (!file_stream.is_open()) { |
| 209 | LOG_ERROR("Can't open input file[%s]", FLAGS_file.c_str()); |
| 210 | return false; |
| 211 | } |
| 212 | |
| 213 | std::string line; |
| 214 | while (std::getline(file_stream, line)) { |
| 215 | line.erase(line.find_last_not_of('\n') + 1); |
| 216 | if (line.empty()) { |
| 217 | continue; |
| 218 | } |
| 219 | std::vector<std::string> res; |
| 220 | ailego::StringHelper::Split(line, ';', &res); |
| 221 | if (res.size() < 2) { |
| 222 | LOG_ERROR("Bad input line, format[key;vector(1 2 3 4...);attributes]"); |
| 223 | continue; |
| 224 | } |
| 225 | |
| 226 | Record record; |
| 227 | // Parse key |
| 228 | uint64_t key = std::stoull(res[0]); |
| 229 | record.key = key; |
| 230 | // Parse feature |
| 231 | if (res.size() >= 2) { |
| 232 | auto data_type = g_collection_meta->index_columns().at(0)->data_type(); |
| 233 | if (data_type == DataTypes::VECTOR_BINARY32) { |
| 234 | std::vector<uint8_t> vec; |
| 235 | ailego::StringHelper::Split(res[1], ' ', &vec); |
| 236 | if (vec.size() == 0 || vec.size() % 32 != 0) { |
| 237 | LOG_ERROR("Bad feature field"); |
| 238 | continue; |
| 239 | } |
| 240 | |
| 241 | std::vector<uint8_t> tmp; |
| 242 | for (size_t i = 0; i < vec.size(); i += 8) { |
| 243 | uint8_t v = 0; |
| 244 | v |= (vec[i] & 0x01) << 7; |
| 245 | v |= (vec[i + 1] & 0x01) << 6; |
| 246 | v |= (vec[i + 2] & 0x01) << 5; |
| 247 | v |= (vec[i + 3] & 0x01) << 4; |
| 248 | v |= (vec[i + 4] & 0x01) << 3; |
| 249 | v |= (vec[i + 5] & 0x01) << 2; |
| 250 | v |= (vec[i + 6] & 0x01) << 1; |
| 251 | v |= (vec[i + 7] & 0x01) << 0; |
| 252 | tmp.push_back(v); |
| 253 | } |
| 254 | |
| 255 | record.vector = |
| 256 | std::string((const char *)tmp.data(), tmp.size() * sizeof(uint8_t)); |
| 257 | record.dimension = vec.size(); |
| 258 | } else { |
| 259 | if (data_type == DataTypes::VECTOR_FP32) { |
| 260 | std::vector<float> feature; |
| 261 | ailego::StringHelper::Split(res[1], ' ', &feature); |
| 262 | if (feature.size() == 0) { |