| 118 | static bvar::LatencyRecorder g_delete_latency_recorder; |
| 119 | |
| 120 | static bool LoadFromTextFile() { |
| 121 | std::ifstream file_stream(FLAGS_file); |
| 122 | if (!file_stream.is_open()) { |
| 123 | LOG_ERROR("Can't open input file %s", FLAGS_file.c_str()); |
| 124 | return false; |
| 125 | } |
| 126 | |
| 127 | std::string line; |
| 128 | while (std::getline(file_stream, line)) { |
| 129 | line.erase(line.find_last_not_of('\n') + 1); |
| 130 | if (line.empty()) { |
| 131 | continue; |
| 132 | } |
| 133 | std::vector<std::string> res; |
| 134 | ailego::StringHelper::Split(line, ';', &res); |
| 135 | if (res.size() < 1) { |
| 136 | LOG_ERROR("Bad input line"); |
| 137 | continue; |
| 138 | } |
| 139 | |
| 140 | Record record; |
| 141 | // Parse key |
| 142 | uint64_t key = std::stoull(res[0]); |
| 143 | record.key = key; |
| 144 | // Parse feature |
| 145 | if (res.size() >= 2) { |
| 146 | if (FLAGS_data_type == "binary") { |
| 147 | std::vector<uint8_t> vec; |
| 148 | ailego::StringHelper::Split(res[1], ' ', &vec); |
| 149 | if (vec.size() == 0 || vec.size() % 32 != 0) { |
| 150 | LOG_ERROR("Bad feature field"); |
| 151 | continue; |
| 152 | } |
| 153 | |
| 154 | std::vector<uint8_t> tmp; |
| 155 | for (size_t i = 0; i < vec.size(); i += 8) { |
| 156 | uint8_t v = 0; |
| 157 | v |= (vec[i] & 0x01) << 7; |
| 158 | v |= (vec[i + 1] & 0x01) << 6; |
| 159 | v |= (vec[i + 2] & 0x01) << 5; |
| 160 | v |= (vec[i + 3] & 0x01) << 4; |
| 161 | v |= (vec[i + 4] & 0x01) << 3; |
| 162 | v |= (vec[i + 5] & 0x01) << 2; |
| 163 | v |= (vec[i + 6] & 0x01) << 1; |
| 164 | v |= (vec[i + 7] & 0x01) << 0; |
| 165 | tmp.push_back(v); |
| 166 | } |
| 167 | |
| 168 | record.vector = |
| 169 | std::string((const char *)tmp.data(), tmp.size() * sizeof(uint8_t)); |
| 170 | record.dimension = vec.size(); |
| 171 | } else { |
| 172 | std::vector<float> feature; |
| 173 | ailego::StringHelper::Split(res[1], ' ', &feature); |
| 174 | if (feature.size() == 0) { |
| 175 | LOG_ERROR("Bad feature field"); |
| 176 | continue; |
| 177 | } |