(path)
| 66 | print("metaDS: Time to read 2048 images: ", time.time() - t) |
| 67 | |
| 68 | def test_metads_dataloader(path): |
| 69 | def process_fn(stream): |
| 70 | for x in stream: |
| 71 | tmp1 = Image.open(io.BytesIO(x['jpg'])).convert('RGB') |
| 72 | # to tensor |
| 73 | tmp1 = torch.from_numpy(np.array(tmp1)).permute(2, 0, 1) |
| 74 | yield tmp1 |
| 75 | |
| 76 | d = MetaDistributedWebDataset(path, process_fn, 0) |
| 77 | batch_size = 1 |
| 78 | loader = torch.utils.data.DataLoader(d, batch_size=batch_size, num_workers=8) |
| 79 | it = iter(loader) |
| 80 | t = time.time() |
| 81 | for i in range(2048 // batch_size): |
| 82 | tmp = next(it) |
| 83 | print("metaDS: Time to read 2048 images: ", time.time() - t) |
| 84 | |
| 85 | def test_metads_resize(path): |
| 86 | def process_fn(stream): |
nothing calls this directly
no test coverage detected