| 156 | |
| 157 | |
| 158 | std::vector<Example> |
| 159 | load_examples(std::vector<std::vector<std::vector<std::string>>> streams) { |
| 160 | ParallelBatchReader reader; |
| 161 | |
| 162 | for (auto& stream : streams) { |
| 163 | if (stream.empty()) |
| 164 | continue; |
| 165 | reader.add(std::make_unique<VectorReader>(std::move(stream))); |
| 166 | } |
| 167 | |
| 168 | const size_t num_examples = reader.num_examples(); |
| 169 | if (num_examples == 0) |
| 170 | return {}; |
| 171 | return reader.get_next(num_examples); |
| 172 | } |
| 173 | |
| 174 | std::vector<Batch> |
| 175 | rebatch_input(const std::vector<Example>& examples, |