| 805 | } |
| 806 | |
| 807 | std::vector<std::string> DatasetLoader::SampleTextDataFromMemory(const std::vector<std::string>& data) { |
| 808 | int sample_cnt = config_.bin_construct_sample_cnt; |
| 809 | if (static_cast<size_t>(sample_cnt) > data.size()) { |
| 810 | sample_cnt = static_cast<int>(data.size()); |
| 811 | } |
| 812 | auto sample_indices = random_.Sample(static_cast<int>(data.size()), sample_cnt); |
| 813 | std::vector<std::string> out(sample_indices.size()); |
| 814 | for (size_t i = 0; i < sample_indices.size(); ++i) { |
| 815 | const size_t idx = sample_indices[i]; |
| 816 | out[i] = data[idx]; |
| 817 | } |
| 818 | return out; |
| 819 | } |
| 820 | |
| 821 | std::vector<std::string> DatasetLoader::SampleTextDataFromFile(const char* filename, const Metadata& metadata, |
| 822 | int rank, int num_machines, int* num_global_data, |