jsonl_path_list: list of jsonl file paths data_dir_list: list of image directories containing the images of each jsonl file num_used_data: list of number of sampled data points for each jsonl
(
self, dataset_name, vit_transform, tokenizer, frame_sampler,
jsonl_path_list, data_dir_list, num_used_data,
local_rank=0, world_size=1, num_workers=8, data_status=None,
shuffle_lines=False, shuffle_seed=0,
)
| 15 | |
| 16 | class SftJSONLIterableDataset(DistributedIterableDataset): |
| 17 | def __init__( |
| 18 | self, dataset_name, vit_transform, tokenizer, frame_sampler, |
| 19 | jsonl_path_list, data_dir_list, num_used_data, |
| 20 | local_rank=0, world_size=1, num_workers=8, data_status=None, |
| 21 | shuffle_lines=False, shuffle_seed=0, |
| 22 | ): |
| 23 | """ |
| 24 | jsonl_path_list: list of jsonl file paths |
| 25 | data_dir_list: list of image directories containing the images of each jsonl file |
| 26 | num_used_data: list of number of sampled data points for each jsonl |
| 27 | """ |
| 28 | super().__init__(dataset_name, local_rank, world_size, num_workers) |
| 29 | self.vit_transform = vit_transform |
| 30 | self.tokenizer = tokenizer |
| 31 | self.frame_sampler = frame_sampler |
| 32 | self.data_status = data_status |
| 33 | self.data_paths = self.get_data_paths( |
| 34 | jsonl_path_list, |
| 35 | data_dir_list, |
| 36 | num_used_data, |
| 37 | shuffle_lines, |
| 38 | shuffle_seed, |
| 39 | ) |
| 40 | self.set_epoch() |
| 41 | |
| 42 | def get_data_paths( |
| 43 | self, |
nothing calls this directly
no test coverage detected