Creates a dataset that reads the given files using the given reader. Args: dataset_creator: A function that takes in a single file name and returns a dataset. filenames: A `tf.data.Dataset` containing one or more filenames. num_parallel_reads: The number of parallel reads we sho
(dataset_creator, filenames, num_parallel_reads=None)
| 65 | |
| 66 | |
| 67 | def _create_dataset_reader(dataset_creator, filenames, num_parallel_reads=None): |
| 68 | """Creates a dataset that reads the given files using the given reader. |
| 69 | |
| 70 | Args: |
| 71 | dataset_creator: A function that takes in a single file name and returns a |
| 72 | dataset. |
| 73 | filenames: A `tf.data.Dataset` containing one or more filenames. |
| 74 | num_parallel_reads: The number of parallel reads we should do. |
| 75 | |
| 76 | Returns: |
| 77 | A `Dataset` that reads data from `filenames`. |
| 78 | """ |
| 79 | def read_one_file(filename): |
| 80 | filename = ops.convert_to_tensor(filename, dtypes.string, name="filename") |
| 81 | return dataset_creator(filename) |
| 82 | |
| 83 | if num_parallel_reads is None: |
| 84 | return filenames.flat_map(read_one_file) |
| 85 | elif num_parallel_reads == dataset_ops.AUTOTUNE: |
| 86 | return filenames.interleave( |
| 87 | read_one_file, num_parallel_calls=num_parallel_reads) |
| 88 | else: |
| 89 | return ParallelInterleaveDataset( |
| 90 | filenames, read_one_file, cycle_length=num_parallel_reads, |
| 91 | block_length=1, sloppy=False, buffer_output_elements=None, |
| 92 | prefetch_input_elements=None) |
| 93 | |
| 94 | |
| 95 | class _TextLineDataset(dataset_ops.DatasetSource): |
no test coverage detected