(shuffling, pad_last_batch)
| 1033 | |
| 1034 | @cartesian_params(("global", "local", "none"), (True, False)) |
| 1035 | def test_shuffling(shuffling, pad_last_batch): |
| 1036 | if not is_gds_supported(): |
| 1037 | raise SkipTest("GDS is not supported in this platform") |
| 1038 | |
| 1039 | with tempfile.TemporaryDirectory(prefix=gds_data_root) as test_data_root: |
| 1040 | # create files |
| 1041 | num_samples = 10 |
| 1042 | batch_size = 3 |
| 1043 | filenames = [] |
| 1044 | for index in range(0, num_samples): |
| 1045 | filename = os.path.join(test_data_root, "test_{:02d}.npy".format(index)) |
| 1046 | filenames.append(filename) |
| 1047 | create_numpy_file(filename, (3, 2, 1), np.int8, False) |
| 1048 | random_shuffle = False |
| 1049 | shuffle_after_epoch = False |
| 1050 | stick_to_shard = False |
| 1051 | if shuffling == "global": |
| 1052 | shuffle_after_epoch = True |
| 1053 | elif shuffle_after_epoch == "local": |
| 1054 | random_shuffle = True |
| 1055 | stick_to_shard = True |
| 1056 | |
| 1057 | pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) |
| 1058 | with pipe: |
| 1059 | data_cpu = fn.readers.numpy( |
| 1060 | device="cpu", |
| 1061 | files=filenames, |
| 1062 | file_root=test_data_root, |
| 1063 | shard_id=0, |
| 1064 | num_shards=2, |
| 1065 | pad_last_batch=pad_last_batch, |
| 1066 | random_shuffle=random_shuffle, |
| 1067 | shuffle_after_epoch=shuffle_after_epoch, |
| 1068 | stick_to_shard=stick_to_shard, |
| 1069 | ) |
| 1070 | data_gpu = fn.readers.numpy( |
| 1071 | device="gpu", |
| 1072 | files=filenames, |
| 1073 | file_root=test_data_root, |
| 1074 | shard_id=0, |
| 1075 | num_shards=2, |
| 1076 | pad_last_batch=pad_last_batch, |
| 1077 | random_shuffle=random_shuffle, |
| 1078 | shuffle_after_epoch=shuffle_after_epoch, |
| 1079 | stick_to_shard=stick_to_shard, |
| 1080 | ) |
| 1081 | pipe.set_outputs(data_cpu, data_gpu) |
| 1082 | |
| 1083 | for _ in range(num_samples // batch_size * 2): |
| 1084 | cpu_arr, gpu_arr = pipe.run() |
| 1085 | assert_array_equal(to_array(cpu_arr), to_array(gpu_arr)) |
| 1086 | |
| 1087 | |
| 1088 | def test_o_direct_alignment(): |
nothing calls this directly
no test coverage detected