Write part of a TFDS sentence dataset to lines in a text file. Args: dataset: tf.dataset containing string-data. maxchars: int: approximate number of characters to save from dataset. data_keys: tuple[str]: what keys in dataset to dump from. Returns: name of temp file with dataset
(dataset: tf.data.Dataset, maxchars: int = int(1e7), data_keys=("text",))
| 42 | |
| 43 | |
| 44 | def _dump_chars_to_textfile(dataset: tf.data.Dataset, maxchars: int = int(1e7), data_keys=("text",)) -> tuple[str, int]: |
| 45 | """Write part of a TFDS sentence dataset to lines in a text file. |
| 46 | Args: |
| 47 | dataset: tf.dataset containing string-data. |
| 48 | maxchars: int: approximate number of characters to save from dataset. |
| 49 | data_keys: tuple[str]: what keys in dataset to dump from. |
| 50 | Returns: |
| 51 | name of temp file with dataset bytes, exact number of characters dumped. |
| 52 | """ |
| 53 | char_count = 0 |
| 54 | ds_iter = dataset.as_numpy_iterator() |
| 55 | temp_dir = tempfile.gettempdir() |
| 56 | with tempfile.NamedTemporaryFile(delete=False, prefix=os.path.join(temp_dir, "ds_chars")) as outfp: |
| 57 | while char_count < maxchars: |
| 58 | example = next(ds_iter) |
| 59 | for k in data_keys: |
| 60 | line = example[k] + b"\n" |
| 61 | char_count += len(line) |
| 62 | outfp.write(line) |
| 63 | return outfp.name, char_count |
| 64 | |
| 65 | |
| 66 | def _train_sentencepiece( |
no outgoing calls
no test coverage detected