| 81 | } |
| 82 | |
| 83 | void Application::LoadData() { |
| 84 | auto start_time = std::chrono::high_resolution_clock::now(); |
| 85 | std::unique_ptr<Predictor> predictor; |
| 86 | // prediction is needed if using input initial model(continued train) |
| 87 | PredictFunction predict_fun = nullptr; |
| 88 | // need to continue training |
| 89 | if (boosting_->NumberOfTotalModel() > 0 && config_.task != TaskType::KRefitTree) { |
| 90 | predictor.reset(new Predictor(boosting_.get(), -1, true, false, false, false, -1, -1)); |
| 91 | predict_fun = predictor->GetPredictFunction(); |
| 92 | } |
| 93 | |
| 94 | // sync up random seed for data partition |
| 95 | if (config_.is_parallel_find_bin) { |
| 96 | config_.data_random_seed = Network::GlobalSyncUpByMin(config_.data_random_seed); |
| 97 | } |
| 98 | |
| 99 | Log::Debug("Loading train file..."); |
| 100 | DatasetLoader dataset_loader(config_, predict_fun, |
| 101 | config_.num_class, config_.data.c_str()); |
| 102 | // load Training data |
| 103 | if (config_.is_parallel_find_bin) { |
| 104 | // load data for parallel training |
| 105 | train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), |
| 106 | config_.initscore_filename.c_str(), |
| 107 | Network::rank(), Network::num_machines())); |
| 108 | } else { |
| 109 | // load data for single machine |
| 110 | train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), config_.initscore_filename.c_str(), |
| 111 | 0, 1)); |
| 112 | } |
| 113 | // need save binary file |
| 114 | if (config_.save_binary) { |
| 115 | train_data_->SaveBinaryFile(nullptr); |
| 116 | } |
| 117 | // create training metric |
| 118 | if (config_.is_provide_training_metric) { |
| 119 | for (auto metric_type : config_.metric) { |
| 120 | auto metric = std::unique_ptr<Metric>(Metric::CreateMetric(metric_type, config_)); |
| 121 | if (metric == nullptr) { continue; } |
| 122 | metric->Init(train_data_->metadata(), train_data_->num_data()); |
| 123 | train_metric_.push_back(std::move(metric)); |
| 124 | } |
| 125 | } |
| 126 | train_metric_.shrink_to_fit(); |
| 127 | |
| 128 | if (!config_.metric.empty()) { |
| 129 | // only when have metrics then need to construct validation data |
| 130 | |
| 131 | // Add validation data, if it exists |
| 132 | for (size_t i = 0; i < config_.valid.size(); ++i) { |
| 133 | Log::Debug("Loading validation file #%zu...", (i + 1)); |
| 134 | // add |
| 135 | auto new_dataset = std::unique_ptr<Dataset>( |
| 136 | dataset_loader.LoadFromFileAlignWithOtherDataset( |
| 137 | config_.valid[i].c_str(), |
| 138 | config_.valid_data_initscores[i].c_str(), |
| 139 | train_data_.get())); |
| 140 | valid_datas_.push_back(std::move(new_dataset)); |
nothing calls this directly
no test coverage detected