Stores records as tfrecord in a fixed number of shards with shuffling.
(self)
| 294 | return writer.finalize() |
| 295 | |
| 296 | def test_write_tfrecord(self): |
| 297 | """Stores records as tfrecord in a fixed number of shards with shuffling.""" |
| 298 | path = os.path.join(self.tmp_dir, 'foo-train.tfrecord') |
| 299 | shards_length, total_size = self._write(to_write=self.RECORDS_TO_WRITE) |
| 300 | self.assertLen(shards_length, self.NUM_SHARDS) |
| 301 | self.assertEqual( |
| 302 | shards_length, [len(shard) for shard in self.SHARDS_CONTENT] |
| 303 | ) |
| 304 | self.assertEqual(total_size, 9) |
| 305 | written_files, all_recs = _read_records(path) |
| 306 | written_index_files, all_indices = _read_indices(path) |
| 307 | self.assertEqual( |
| 308 | written_files, |
| 309 | [ |
| 310 | f'foo-train.tfrecord-{i:05d}-of-{self.NUM_SHARDS:05d}' |
| 311 | for i in range(self.NUM_SHARDS) |
| 312 | if shards_length[i] |
| 313 | ], |
| 314 | ) |
| 315 | self.assertEqual(all_recs, self.SHARDS_CONTENT) |
| 316 | self.assertEmpty(written_index_files) |
| 317 | self.assertEmpty(all_indices) |
| 318 | |
| 319 | def test_write_tfrecord_sorted_by_key(self): |
| 320 | """Stores records as tfrecord in a fixed number of shards without shuffling.""" |
nothing calls this directly
no test coverage detected