(
data_args,
output_dir, # NOTE: from training_args.output_dir
vla_processor=None,
)
| 173 | |
| 174 | |
| 175 | def build_datasets( |
| 176 | data_args, |
| 177 | output_dir, # NOTE: from training_args.output_dir |
| 178 | vla_processor=None, |
| 179 | ) -> IterableDataset: |
| 180 | train_dataset = OpenXIterableDataset( |
| 181 | data_args.data_root_dir, |
| 182 | output_dir, |
| 183 | data_args.data_mix, |
| 184 | is_train=True, |
| 185 | max_length=data_args.max_seq_length, |
| 186 | shuffle_buffer_size=data_args.shuffle_buffer_size, |
| 187 | tsfm_thread_muti=data_args.tsfm_thread_muti, |
| 188 | read_thread_muti=data_args.read_thread_muti, |
| 189 | obs_backward_steps=data_args.obs_backward_steps, |
| 190 | obs_backward_delta=data_args.obs_backward_delta, |
| 191 | action_forward_steps=data_args.action_forward_steps, |
| 192 | use_raw_dataloader=data_args.use_raw_dataloader, |
| 193 | fix_raw_length=data_args.fix_raw_length, |
| 194 | vla_processor=vla_processor, |
| 195 | ) |
| 196 | eval_dataset = None |
| 197 | return train_dataset, eval_dataset |
no test coverage detected