! \brief Extract local features from file */
| 1139 | |
| 1140 | /*! \brief Extract local features from file */ |
| 1141 | void DatasetLoader::ExtractFeaturesFromFile(const char* filename, const Parser* parser, |
| 1142 | const std::vector<data_size_t>& used_data_indices, Dataset* dataset) { |
| 1143 | std::vector<double> init_score; |
| 1144 | if (predict_fun_ != nullptr) { |
| 1145 | init_score = std::vector<double>(dataset->num_data_ * num_class_); |
| 1146 | } |
| 1147 | std::function<void(data_size_t, const std::vector<std::string>&)> process_fun = |
| 1148 | [this, &init_score, &parser, &dataset] |
| 1149 | (data_size_t start_idx, const std::vector<std::string>& lines) { |
| 1150 | std::vector<std::pair<int, double>> oneline_features; |
| 1151 | double tmp_label = 0.0f; |
| 1152 | OMP_INIT_EX(); |
| 1153 | #pragma omp parallel for schedule(static) private(oneline_features) firstprivate(tmp_label) |
| 1154 | for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); ++i) { |
| 1155 | OMP_LOOP_EX_BEGIN(); |
| 1156 | const int tid = omp_get_thread_num(); |
| 1157 | oneline_features.clear(); |
| 1158 | // parser |
| 1159 | parser->ParseOneLine(lines[i].c_str(), &oneline_features, &tmp_label); |
| 1160 | // set initial score |
| 1161 | if (!init_score.empty()) { |
| 1162 | std::vector<double> oneline_init_score(num_class_); |
| 1163 | predict_fun_(oneline_features, oneline_init_score.data()); |
| 1164 | for (int k = 0; k < num_class_; ++k) { |
| 1165 | init_score[k * dataset->num_data_ + start_idx + i] = static_cast<double>(oneline_init_score[k]); |
| 1166 | } |
| 1167 | } |
| 1168 | // set label |
| 1169 | dataset->metadata_.SetLabelAt(start_idx + i, static_cast<label_t>(tmp_label)); |
| 1170 | // push data |
| 1171 | for (auto& inner_data : oneline_features) { |
| 1172 | if (inner_data.first >= dataset->num_total_features_) { continue; } |
| 1173 | int feature_idx = dataset->used_feature_map_[inner_data.first]; |
| 1174 | if (feature_idx >= 0) { |
| 1175 | // if is used feature |
| 1176 | int group = dataset->feature2group_[feature_idx]; |
| 1177 | int sub_feature = dataset->feature2subfeature_[feature_idx]; |
| 1178 | dataset->feature_groups_[group]->PushData(tid, sub_feature, start_idx + i, inner_data.second); |
| 1179 | } else { |
| 1180 | if (inner_data.first == weight_idx_) { |
| 1181 | dataset->metadata_.SetWeightAt(start_idx + i, static_cast<label_t>(inner_data.second)); |
| 1182 | } else if (inner_data.first == group_idx_) { |
| 1183 | dataset->metadata_.SetQueryAt(start_idx + i, static_cast<data_size_t>(inner_data.second)); |
| 1184 | } |
| 1185 | } |
| 1186 | } |
| 1187 | OMP_LOOP_EX_END(); |
| 1188 | } |
| 1189 | OMP_THROW_EX(); |
| 1190 | }; |
| 1191 | TextReader<data_size_t> text_reader(filename, config_.header, config_.file_load_progress_interval_bytes); |
| 1192 | if (!used_data_indices.empty()) { |
| 1193 | // only need part of data |
| 1194 | text_reader.ReadPartAndProcessParallel(used_data_indices, process_fun); |
| 1195 | } else { |
| 1196 | // need full data |
| 1197 | text_reader.ReadAllAndProcessParallel(process_fun); |
| 1198 | } |
nothing calls this directly
no test coverage detected