Adds an example to the shard. If there is no open shard, it starts a new one. Args: serialized_example: example to add to the shard. Raises: ValueError: if the split is already closed.
(self, serialized_example: str)
| 82 | file_format: file_adapters.FileFormat = file_adapters.FileFormat.TFRECORD |
| 83 | |
| 84 | def add_example(self, serialized_example: str) -> None: |
| 85 | """Adds an example to the shard. |
| 86 | |
| 87 | If there is no open shard, it starts a new one. |
| 88 | |
| 89 | Args: |
| 90 | serialized_example: example to add to the shard. |
| 91 | |
| 92 | Raises: |
| 93 | ValueError: if the split is already closed. |
| 94 | """ |
| 95 | if self.closed: |
| 96 | raise ValueError(f'Split {self.info.name} is already closed.') |
| 97 | if self.current_shard is None: |
| 98 | if self.info.filename_template is None: |
| 99 | raise ValueError(f'Split {self.info.name} has no filename template.') |
| 100 | path = self.info.filename_template.sharded_filepath( |
| 101 | shard_index=self.complete_shards, num_shards=None |
| 102 | ) |
| 103 | if self.file_format == file_adapters.FileFormat.TFRECORD: |
| 104 | self.current_shard = Shard(writer=tf.io.TFRecordWriter(os.fspath(path))) |
| 105 | elif self.file_format == file_adapters.FileFormat.ARRAY_RECORD: |
| 106 | self.current_shard = Shard( |
| 107 | writer=array_record_module.ArrayRecordWriter( |
| 108 | os.fspath(path), 'group_size:1' |
| 109 | ) |
| 110 | ) |
| 111 | else: |
| 112 | raise ValueError('Unknown file format %s' % self.file_format) |
| 113 | self.current_shard.add_example(serialized_example) |
| 114 | |
| 115 | def close_shard(self) -> None: |
| 116 | """Finalizes a shard and updates the split metadata accordingly.""" |
nothing calls this directly
no test coverage detected