Function
file_reader_pipeline
(
files,
exts=None,
dont_use_mmap=False,
num_shards=1,
shard_id=0,
skip_cached_images=False,
pad_last_batch=False,
lazy_init=False,
read_ahead=False,
stick_to_shard=False,
)
Source from the content-addressed store, hash-verified
| 69 | |
| 70 | @pipeline_def() |
| 71 | def file_reader_pipeline( |
| 72 | files, |
| 73 | exts=None, |
| 74 | dont_use_mmap=False, |
| 75 | num_shards=1, |
| 76 | shard_id=0, |
| 77 | skip_cached_images=False, |
| 78 | pad_last_batch=False, |
| 79 | lazy_init=False, |
| 80 | read_ahead=False, |
| 81 | stick_to_shard=False, |
| 82 | ): |
| 83 | if not isinstance(exts, list): |
| 84 | exts = [exts] |
| 85 | |
| 86 | return tuple( |
| 87 | ( |
| 88 | readers.file( |
| 89 | files=filter_ext(files, ext), |
| 90 | dont_use_mmap=dont_use_mmap, |
| 91 | prefetch_queue_depth=1, |
| 92 | num_shards=num_shards, |
| 93 | shard_id=shard_id, |
| 94 | stick_to_shard=stick_to_shard, |
| 95 | skip_cached_images=skip_cached_images, |
| 96 | pad_last_batch=pad_last_batch, |
| 97 | lazy_init=lazy_init, |
| 98 | read_ahead=read_ahead, |
| 99 | )[0] |
| 100 | if type(ext) in {str, set} |
| 101 | else ext |
| 102 | ) |
| 103 | for ext in exts |
| 104 | ) |
| 105 | |
| 106 | |
| 107 | def generate_temp_index_file(tar_file_path): |