(
path,
batch_size,
device="cpu",
file_list=None,
files=None,
file_filter="*.npy",
num_threads=1,
device_id=0,
cache_header_information=False,
pad_last_batch=False,
dont_use_mmap=False,
enable_o_direct=False,
shard_id=0,
num_shards=1,
)
| 80 | |
| 81 | |
| 82 | def NumpyReaderPipeline( |
| 83 | path, |
| 84 | batch_size, |
| 85 | device="cpu", |
| 86 | file_list=None, |
| 87 | files=None, |
| 88 | file_filter="*.npy", |
| 89 | num_threads=1, |
| 90 | device_id=0, |
| 91 | cache_header_information=False, |
| 92 | pad_last_batch=False, |
| 93 | dont_use_mmap=False, |
| 94 | enable_o_direct=False, |
| 95 | shard_id=0, |
| 96 | num_shards=1, |
| 97 | ): |
| 98 | pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id) |
| 99 | data = fn.readers.numpy( |
| 100 | device=device, |
| 101 | file_list=file_list, |
| 102 | files=files, |
| 103 | file_root=path, |
| 104 | file_filter=file_filter, |
| 105 | shard_id=shard_id, |
| 106 | num_shards=num_shards, |
| 107 | cache_header_information=cache_header_information, |
| 108 | pad_last_batch=pad_last_batch, |
| 109 | dont_use_mmap=dont_use_mmap, |
| 110 | use_o_direct=enable_o_direct, |
| 111 | ) |
| 112 | pipe.set_outputs(data) |
| 113 | return pipe |
| 114 | |
| 115 | |
| 116 | all_numpy_types = set( |
no test coverage detected