()
| 211 | |
| 212 | |
| 213 | def test_sharding(): |
| 214 | num_samples = 1000 |
| 215 | tar_file_path = os.path.join(get_dali_extra_path(), "db/webdataset/MNIST/devel-0.tar") |
| 216 | index_file = generate_temp_index_file(tar_file_path) |
| 217 | |
| 218 | extract_dir = generate_temp_extract(tar_file_path) |
| 219 | equivalent_files = sorted( |
| 220 | glob(extract_dir.name + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) |
| 221 | ) # noqa: 203 |
| 222 | |
| 223 | num_shards = 100 |
| 224 | for shard_id in range(num_shards): |
| 225 | compare_pipelines( |
| 226 | webdataset_raw_pipeline( |
| 227 | tar_file_path, |
| 228 | index_file.name, |
| 229 | ["jpg", "cls"], |
| 230 | num_shards=num_shards, |
| 231 | shard_id=shard_id, |
| 232 | batch_size=test_batch_size, |
| 233 | device_id=0, |
| 234 | num_threads=1, |
| 235 | ), |
| 236 | file_reader_pipeline( |
| 237 | equivalent_files, |
| 238 | ["jpg", "cls"], |
| 239 | num_shards=num_shards, |
| 240 | shard_id=shard_id, |
| 241 | batch_size=test_batch_size, |
| 242 | device_id=0, |
| 243 | num_threads=1, |
| 244 | ), |
| 245 | test_batch_size, |
| 246 | math.ceil(num_samples / num_shards / test_batch_size) * 2, |
| 247 | ) |
| 248 | |
| 249 | |
| 250 | def test_pax_format(): |
nothing calls this directly
no test coverage detected