(args, preprocess_fn, is_train, epoch=0, tokenizer=None)
| 468 | |
| 469 | |
| 470 | def get_csv_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): |
| 471 | input_filename = args.train_data if is_train else args.val_data |
| 472 | assert input_filename |
| 473 | dataset = CsvDataset( |
| 474 | input_filename, |
| 475 | preprocess_fn, |
| 476 | img_key=args.csv_img_key, |
| 477 | caption_key=args.csv_caption_key, |
| 478 | sep=args.csv_separator, |
| 479 | tokenizer=tokenizer |
| 480 | ) |
| 481 | num_samples = len(dataset) |
| 482 | sampler = DistributedSampler( |
| 483 | dataset) if args.distributed and is_train else None |
| 484 | shuffle = is_train and sampler is None |
| 485 | |
| 486 | dataloader = DataLoader( |
| 487 | dataset, |
| 488 | batch_size=args.batch_size, |
| 489 | shuffle=shuffle, |
| 490 | num_workers=args.workers, |
| 491 | pin_memory=True, |
| 492 | sampler=sampler, |
| 493 | drop_last=is_train, |
| 494 | ) |
| 495 | dataloader.num_samples = num_samples |
| 496 | dataloader.num_batches = len(dataloader) |
| 497 | |
| 498 | return DataInfo(dataloader, sampler) |
| 499 | |
| 500 | |
| 501 | class SyntheticDataset(Dataset): |
nothing calls this directly
no test coverage detected