(data_name)
| 113 | |
| 114 | # Preprocesses a dataset |
| 115 | def preprocess_dataset(data_name): |
| 116 | input_dir = get_data_dir(cfg, data_name) |
| 117 | print('\nDataset:', input_dir) |
| 118 | if not os.path.isdir(input_dir): |
| 119 | print('Not found') |
| 120 | return |
| 121 | |
| 122 | # Create the output directory |
| 123 | output_name = data_name + '.' + WORKER_UID |
| 124 | output_dir = os.path.join(cfg.preproc_dir, output_name) |
| 125 | os.makedirs(output_dir) |
| 126 | |
| 127 | # Preprocess image sample groups |
| 128 | sample_groups = get_image_sample_groups(input_dir, input_features, target_features) |
| 129 | tza_filename = os.path.join(output_dir, 'images.tza') |
| 130 | samples = [] |
| 131 | with tza.Writer(tza_filename) as output_tza: |
| 132 | for _, input_names, target_name in sample_groups: |
| 133 | if target_name: |
| 134 | samples += preprocess_sample_group(input_dir, output_tza, input_names, target_name) |
| 135 | |
| 136 | # Save the samples in the dataset |
| 137 | samples_filename = os.path.join(output_dir, 'samples.json') |
| 138 | save_json(samples_filename, samples) |
| 139 | |
| 140 | # Save the config |
| 141 | save_config(output_dir, cfg) |
| 142 | |
| 143 | # Preprocess all datasets |
| 144 | with torch.inference_mode(): |
no test coverage detected