Finalizes a shard and updates the split metadata accordingly.
(self)
| 113 | self.current_shard.add_example(serialized_example) |
| 114 | |
| 115 | def close_shard(self) -> None: |
| 116 | """Finalizes a shard and updates the split metadata accordingly.""" |
| 117 | if not self.current_shard: |
| 118 | return |
| 119 | self.current_shard.close_writer() |
| 120 | self.info = splits_lib.SplitInfo( |
| 121 | name=self.info.name, |
| 122 | shard_lengths=self.info.shard_lengths |
| 123 | + [self.current_shard.num_examples], |
| 124 | num_bytes=self.info.num_bytes + self.current_shard.num_bytes, |
| 125 | filename_template=self.info.filename_template, |
| 126 | ) |
| 127 | self.complete_shards += 1 |
| 128 | self.current_shard = None |
| 129 | |
| 130 | def close(self) -> None: |
| 131 | """Closes the split (and the shard if it is still open).""" |
no test coverage detected