| 38 | public: |
| 39 | OF_DISALLOW_COPY_AND_MOVE(BatchReader); |
| 40 | BatchReader(std::vector<std::unique_ptr<embedding::PosixFile>>&& files, |
| 41 | std::vector<Block>&& blocks, size_t block_size_bytes, size_t num_workers) |
| 42 | : head_(0), |
| 43 | tail_(0), |
| 44 | files_(std::move(files)), |
| 45 | blocks_(blocks), |
| 46 | block_size_bytes_(block_size_bytes), |
| 47 | num_workers_(num_workers) { |
| 48 | for (size_t i = 0; i < num_workers_; ++i) { |
| 49 | Worker worker; |
| 50 | auto* sq = new Channel<BatchReaderRequest>(); |
| 51 | auto* cq = new Channel<BatchReaderRequest>(); |
| 52 | worker.sq.reset(sq); |
| 53 | worker.cq.reset(cq); |
| 54 | worker.thread = std::thread([sq, cq, this]() { |
| 55 | while (true) { |
| 56 | BatchReaderRequest request; |
| 57 | auto status = sq->Receive(&request); |
| 58 | if (status == kChannelStatusErrorClosed) { break; } |
| 59 | CHECK_EQ(status, kChannelStatusSuccess) << "channel error"; |
| 60 | size_t buffer_offset = 0; |
| 61 | for (size_t i = 0; i < request.blocks->size(); ++i) { |
| 62 | size_t block_index = request.blocks->at(i); |
| 63 | const Block& block = blocks_[block_index]; |
| 64 | size_t remaining = block_size_bytes_; |
| 65 | size_t file_index = block.file_index; |
| 66 | size_t file_offset = block.offset_in_file; |
| 67 | while (remaining != 0) { |
| 68 | const size_t bytes_to_read = |
| 69 | std::min(remaining, files_.at(file_index)->Size() - file_offset); |
| 70 | PCHECK(pread(files_[file_index]->fd(), |
| 71 | reinterpret_cast<unsigned char*>(request.buffer) + buffer_offset, |
| 72 | bytes_to_read, file_offset) |
| 73 | == bytes_to_read) |
| 74 | << "file read error"; |
| 75 | remaining -= bytes_to_read; |
| 76 | buffer_offset += bytes_to_read; |
| 77 | if (remaining != 0) { |
| 78 | file_index = (file_index + 1) % files_.size(); |
| 79 | file_offset = 0; |
| 80 | } |
| 81 | } |
| 82 | } |
| 83 | CHECK(cq->Send(std::move(request)) == kChannelStatusSuccess) << "channel error"; |
| 84 | } |
| 85 | }); |
| 86 | workers_.emplace_back(std::move(worker)); |
| 87 | } |
| 88 | } |
| 89 | ~BatchReader() { |
| 90 | for (auto& work : workers_) { work.Close(); } |
| 91 | } |