| 166 | } |
| 167 | |
| 168 | Dataset* DatasetLoader::LoadFromFile(const char* filename, const char* initscore_file, int rank, int num_machines) { |
| 169 | // don't support query id in data file when training in parallel |
| 170 | if (num_machines > 1 && !config_.pre_partition) { |
| 171 | if (group_idx_ > 0) { |
| 172 | Log::Fatal("Using a query id without pre-partitioning the data file is not supported for parallel training.\n" |
| 173 | "Please use an additional query file or pre-partition the data"); |
| 174 | } |
| 175 | } |
| 176 | auto dataset = std::unique_ptr<Dataset>(new Dataset()); |
| 177 | data_size_t num_global_data = 0; |
| 178 | std::vector<data_size_t> used_data_indices; |
| 179 | auto bin_filename = CheckCanLoadFromBin(filename); |
| 180 | if (bin_filename.size() == 0) { |
| 181 | auto parser = std::unique_ptr<Parser>(Parser::CreateParser(filename, config_.header, 0, label_idx_)); |
| 182 | if (parser == nullptr) { |
| 183 | Log::Fatal("Could not recognize data format of %s", filename); |
| 184 | } |
| 185 | dataset->data_filename_ = filename; |
| 186 | dataset->label_idx_ = label_idx_; |
| 187 | dataset->metadata_.Init(filename, initscore_file); |
| 188 | if (!config_.two_round) { |
| 189 | // read data to memory |
| 190 | auto text_data = LoadTextDataToMemory(filename, dataset->metadata_, rank, num_machines, &num_global_data, &used_data_indices); |
| 191 | dataset->num_data_ = static_cast<data_size_t>(text_data.size()); |
| 192 | // sample data |
| 193 | auto sample_data = SampleTextDataFromMemory(text_data); |
| 194 | // construct feature bin mappers |
| 195 | ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get()); |
| 196 | // initialize label |
| 197 | dataset->metadata_.Init(dataset->num_data_, weight_idx_, group_idx_); |
| 198 | // extract features |
| 199 | ExtractFeaturesFromMemory(&text_data, parser.get(), dataset.get()); |
| 200 | text_data.clear(); |
| 201 | } else { |
| 202 | // sample data from file |
| 203 | auto sample_data = SampleTextDataFromFile(filename, dataset->metadata_, rank, num_machines, &num_global_data, &used_data_indices); |
| 204 | if (used_data_indices.size() > 0) { |
| 205 | dataset->num_data_ = static_cast<data_size_t>(used_data_indices.size()); |
| 206 | } else { |
| 207 | dataset->num_data_ = num_global_data; |
| 208 | } |
| 209 | // construct feature bin mappers |
| 210 | ConstructBinMappersFromTextData(rank, num_machines, sample_data, parser.get(), dataset.get()); |
| 211 | // initialize label |
| 212 | dataset->metadata_.Init(dataset->num_data_, weight_idx_, group_idx_); |
| 213 | Log::Debug("Making second pass..."); |
| 214 | // extract features |
| 215 | ExtractFeaturesFromFile(filename, parser.get(), used_data_indices, dataset.get()); |
| 216 | } |
| 217 | } else { |
| 218 | // load data from binary file |
| 219 | dataset.reset(LoadFromBinFile(filename, bin_filename.c_str(), rank, num_machines, &num_global_data, &used_data_indices)); |
| 220 | } |
| 221 | // check meta data |
| 222 | dataset->metadata_.CheckOrPartition(num_global_data, used_data_indices); |
| 223 | // need to check training data |
| 224 | CheckDataset(dataset.get()); |
| 225 | return dataset.release(); |