(path)
| 83 | print("metaDS: Time to read 2048 images: ", time.time() - t) |
| 84 | |
| 85 | def test_metads_resize(path): |
| 86 | def process_fn(stream): |
| 87 | for x in stream: |
| 88 | tmp1 = Image.open(io.BytesIO(x['jpg'])).convert('RGB').resize((256, 256)) |
| 89 | # to tensor |
| 90 | tmp1 = torch.from_numpy(np.array(tmp1)).permute(2, 0, 1) |
| 91 | # resize |
| 92 | # tmp1 = torch.nn.functional.interpolate(tmp1.unsqueeze(0), size=(256, 256), mode='bilinear', align_corners=False) |
| 93 | yield tmp1 |
| 94 | |
| 95 | d = MetaDistributedWebDataset(path, process_fn, 0) |
| 96 | batch_size = 1 |
| 97 | loader = torch.utils.data.DataLoader(d, batch_size=batch_size, num_workers=4) |
| 98 | it = iter(loader) |
| 99 | t = time.time() |
| 100 | for i in range(2048 // batch_size): |
| 101 | tmp = next(it) |
| 102 | print("metaDS: Time to read 2048 images: ", time.time() - t) |
| 103 | |
| 104 | |
| 105 | def test_metads_batch(path): |
nothing calls this directly
no test coverage detected