(self, config, mode, logger, seed=None, epoch=1, task='rec')
| 9 | class TextLMDBDataSet(Dataset): |
| 10 | |
| 11 | def __init__(self, config, mode, logger, seed=None, epoch=1, task='rec'): |
| 12 | super(TextLMDBDataSet, self).__init__() |
| 13 | |
| 14 | global_config = config['Global'] |
| 15 | dataset_config = config[mode]['dataset'] |
| 16 | loader_config = config[mode]['loader'] |
| 17 | loader_config['batch_size_per_card'] |
| 18 | data_dir = dataset_config['data_dir'] |
| 19 | self.do_shuffle = loader_config['shuffle'] |
| 20 | |
| 21 | self.lmdb_sets = self.load_hierarchical_lmdb_dataset(data_dir) |
| 22 | logger.info(f'Initialize indexs of datasets: {data_dir}') |
| 23 | self.data_idx_order_list = self.dataset_traversal() |
| 24 | if self.do_shuffle: |
| 25 | np.random.shuffle(self.data_idx_order_list) |
| 26 | self.ops = create_operators(dataset_config['transforms'], |
| 27 | global_config) |
| 28 | self.ext_op_transform_idx = dataset_config.get('ext_op_transform_idx', |
| 29 | 1) |
| 30 | |
| 31 | ratio_list = dataset_config.get('ratio_list', [1.0]) |
| 32 | self.need_reset = True in [x < 1 for x in ratio_list] |
| 33 | |
| 34 | def load_hierarchical_lmdb_dataset(self, data_dir): |
| 35 | lmdb_sets = {} |
nothing calls this directly
no test coverage detected