(
self,
tokenizer: transformers.PreTrainedTokenizer,
data_path: str,
data_args
)
| 161 | """Dataset for supervised fine-tuning.""" |
| 162 | |
| 163 | def __init__( |
| 164 | self, |
| 165 | tokenizer: transformers.PreTrainedTokenizer, |
| 166 | data_path: str, |
| 167 | data_args |
| 168 | ): |
| 169 | super(LazySupervisedMixDataset, self).__init__() |
| 170 | |
| 171 | self.data_args = data_args |
| 172 | list_data_dict = [] |
| 173 | |
| 174 | |
| 175 | cache_dir_path='../cache' |
| 176 | |
| 177 | |
| 178 | # blip3o |
| 179 | |
| 180 | # sft |
| 181 | jpg_files=[ |
| 182 | '../BLIP3o/SFT_60K_data/processed/*.tar', |
| 183 | ] |
| 184 | |
| 185 | train_dataset = load_dataset("webdataset", data_files=jpg_files, split="train", num_proc=32, cache_dir=cache_dir_path) |
| 186 | train_dataset = train_dataset.rename_column("jpg", "image") |
| 187 | |
| 188 | train_dataset = train_dataset.add_column('type', len(train_dataset) * ['T2I']) |
| 189 | train_dataset = train_dataset.remove_columns([col for col in train_dataset.column_names if not col in ( |
| 190 | ["image", "txt", "type"])]) |
| 191 | print(f"finish loading image {len(train_dataset)}") |
| 192 | list_data_dict.append(train_dataset) |
| 193 | |
| 194 | |
| 195 | # # Echo-4o-Image |
| 196 | data_root = "../Echo-4o-Image/Instruction-Following-Image/images-processed' |
| 197 | tar_list = sorted(glob.glob(f'{data_root}/*.tar.gz')) |
| 198 | train_dataset = load_dataset("webdataset", data_files=tar_list, split="train", num_proc=32, cache_dir=cache_dir_path) |
| 199 | train_dataset = train_dataset.rename_column("jpg", "image") |
| 200 | |
| 201 | train_dataset = train_dataset.add_column('type', len(train_dataset) * ['T2I']) |
| 202 | train_dataset = train_dataset.remove_columns([col for col in train_dataset.column_names if not col in ( |
| 203 | ["image", "txt", "type"])]) |
| 204 | print(f"finish loading image {len(train_dataset)}") |
| 205 | list_data_dict.append(train_dataset) |
| 206 | |
| 207 | |
| 208 | |
| 209 | # # sharegpt |
| 210 | train_dataset = load_dataset("webdataset", data_files='../FreedomIntelligence/ShareGPT-4o-Image-processed/text_to_image_part_*.tar', split="train", num_proc=32, cache_dir=cache_dir_path) |
| 211 | train_dataset = train_dataset.rename_column("png", "image") |
| 212 | |
| 213 | train_dataset = train_dataset.add_column('type', len(train_dataset) * ['T2I']) |
| 214 | train_dataset = train_dataset.remove_columns([col for col in train_dataset.column_names if not col in ( |
| 215 | ["image", "txt", "type"])]) |
| 216 | print(f"finish loading image {len(train_dataset)}") |
| 217 | list_data_dict.append(train_dataset) |
| 218 | |
| 219 | |
| 220 |
nothing calls this directly
no test coverage detected