(self, config, mode, logger, seed=None, epoch=1, task='rec')
| 35 | class RatioDataSetTest(Dataset): |
| 36 | |
| 37 | def __init__(self, config, mode, logger, seed=None, epoch=1, task='rec'): |
| 38 | super(RatioDataSetTest, self).__init__() |
| 39 | self.ds_width = config[mode]['dataset'].get('ds_width', True) |
| 40 | global_config = config['Global'] |
| 41 | dataset_config = config[mode]['dataset'] |
| 42 | loader_config = config[mode]['loader'] |
| 43 | max_ratio = loader_config.get('max_ratio', 10) |
| 44 | min_ratio = loader_config.get('min_ratio', 1) |
| 45 | data_dir_list = dataset_config['data_dir_list'] |
| 46 | self.do_shuffle = loader_config['shuffle'] |
| 47 | self.seed = epoch |
| 48 | self.max_text_length = global_config['max_text_length'] |
| 49 | data_source_num = len(data_dir_list) |
| 50 | ratio_list = dataset_config.get('ratio_list', 1.0) |
| 51 | if isinstance(ratio_list, (float, int)): |
| 52 | ratio_list = [float(ratio_list)] * int(data_source_num) |
| 53 | assert len( |
| 54 | ratio_list |
| 55 | ) == data_source_num, 'The length of ratio_list should be the same as the file_list.' |
| 56 | self.lmdb_sets = self.load_hierarchical_lmdb_dataset( |
| 57 | data_dir_list, ratio_list) |
| 58 | for data_dir in data_dir_list: |
| 59 | logger.info('Initialize indexs of datasets:%s' % data_dir) |
| 60 | self.logger = logger |
| 61 | data_idx_order_list = self.dataset_traversal() |
| 62 | character_dict_path = global_config.get('character_dict_path', None) |
| 63 | use_space_char = global_config.get('use_space_char', False) |
| 64 | if character_dict_path is None: |
| 65 | char_test = '0123456789abcdefghijklmnopqrstuvwxyz' |
| 66 | else: |
| 67 | char_test = '' |
| 68 | with open(character_dict_path, 'rb') as fin: |
| 69 | lines = fin.readlines() |
| 70 | for line in lines: |
| 71 | line = line.decode('utf-8').strip('\n').strip('\r\n') |
| 72 | char_test += line |
| 73 | if use_space_char: |
| 74 | char_test += ' ' |
| 75 | wh_ratio, data_idx_order_list = self.get_wh_ratio( |
| 76 | data_idx_order_list, char_test) |
| 77 | self.data_idx_order_list = np.array(data_idx_order_list) |
| 78 | wh_ratio = np.around(np.array(wh_ratio)) |
| 79 | self.wh_ratio = np.clip(wh_ratio, a_min=min_ratio, a_max=max_ratio) |
| 80 | for i in range(max_ratio + 1): |
| 81 | logger.info((1 * (self.wh_ratio == i)).sum()) |
| 82 | self.wh_ratio_sort = np.argsort(self.wh_ratio) |
| 83 | self.ops = create_operators(dataset_config['transforms'], |
| 84 | global_config) |
| 85 | |
| 86 | self.need_reset = True in [x < 1 for x in ratio_list] |
| 87 | self.error = 0 |
| 88 | self.base_shape = dataset_config.get( |
| 89 | 'base_shape', [[64, 64], [96, 48], [112, 40], [128, 32]]) |
| 90 | self.base_h = 32 |
| 91 | |
| 92 | def get_wh_ratio(self, data_idx_order_list, char_test): |
| 93 | wh_ratio = [] |
nothing calls this directly
no test coverage detected