| 29 | } |
| 30 | |
| 31 | int SlotReader::read(ExampleInfo* info) { |
| 32 | CHECK_GT(FLAGS_num_threads, 0); |
| 33 | { |
| 34 | Lock l(mu_); |
| 35 | loaded_file_count_ = 0; |
| 36 | num_ex_.resize(data_.file_size()); |
| 37 | } |
| 38 | { |
| 39 | if (FLAGS_verbose) { |
| 40 | for (size_t i = 0; i < data_.file_size(); ++i) { |
| 41 | LI << "I will load data file [" << i + 1 << "/" << |
| 42 | data_.file_size() << "] [" << data_.file(i) << "]"; |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | ThreadPool pool(FLAGS_num_threads); |
| 47 | for (int i = 0; i < data_.file_size(); ++i) { |
| 48 | auto one_file = ithFile(data_, i); |
| 49 | pool.add([this, one_file, i](){ readOneFile(one_file, i); }); |
| 50 | } |
| 51 | pool.startWorkers(); |
| 52 | } |
| 53 | if (info) *info = info_; |
| 54 | for (int i = 0; i < info_.slot_size(); ++i) { |
| 55 | slot_info_[info_.slot(i).id()] = info_.slot(i); |
| 56 | } |
| 57 | return 0; |
| 58 | } |
| 59 | |
| 60 | bool SlotReader::readOneFile(const DataConfig& data, int ith_file) { |
| 61 | if (FLAGS_verbose) { |