| 136 | |
| 137 | |
| 138 | class DataLoader(object): |
| 139 | def __init__(self, input_var, file_list, config): |
| 140 | assert isinstance(input_var, list) |
| 141 | assert len(file_list) > 0 |
| 142 | self.input_var = input_var |
| 143 | self.file_list = file_list |
| 144 | self.config = config |
| 145 | |
| 146 | def get_reader(self): |
| 147 | logger.info("Get DataLoader") |
| 148 | loader = paddle.io.DataLoader.from_generator( |
| 149 | feed_list=self.input_var, |
| 150 | capacity=64, |
| 151 | iterable=False, |
| 152 | use_double_buffer=False) |
| 153 | path = self.config.get("runner.train_reader_path") |
| 154 | path = os.path.join(self.config["config_abs_dir"], path) |
| 155 | generator = get_reader_generator(path) |
| 156 | generator.init(self.config) |
| 157 | batch_size = int(self.config.get("runner.train_batch_size")) |
| 158 | batch_generator = self.config.get("runner.batch_generator", False) |
| 159 | if batch_generator: |
| 160 | loader.set_batch_generator(generator.dataloader(self.file_list)) |
| 161 | else: |
| 162 | loader.set_sample_generator( |
| 163 | generator.dataloader(self.file_list), |
| 164 | batch_size=batch_size, |
| 165 | drop_last=True, |
| 166 | places=paddle.static.cpu_places()) |
| 167 | return loader |
| 168 | |
| 169 | |
| 170 | class InferDataLoader(object): |
no outgoing calls
no test coverage detected