(self)
| 220 | assert self.thread_num >= 1 |
| 221 | |
| 222 | def get_reader(self): |
| 223 | logger.info("Get Dataset") |
| 224 | dataset = paddle.distributed.QueueDataset() |
| 225 | dataset.init( |
| 226 | use_var=self.input_var, |
| 227 | pipe_command=self.pipe_command, |
| 228 | batch_size=self.batch_size, |
| 229 | thread_num=self.thread_num) |
| 230 | print("dataset get_reader thread_num:", self.thread_num) |
| 231 | dataset.set_filelist(self.file_list) |
| 232 | return dataset |
| 233 | |
| 234 | |
| 235 | class InmemoryDatset(object): |