Tokenizes output_features using SeqIO. Args: output_features: A mapping from field name to seqio.preprocessors.OutputFeaturesType. Their vocabulary attribute will be used to tokenize the specified feature. Alternatively, a config which instantiates to one.
(
output_features: ConfigOr[seqio.preprocessors.OutputFeaturesType],
copy_pretokenized: bool = True,
with_eos: bool = False,
)
| 132 | |
| 133 | |
| 134 | def tokenize( |
| 135 | output_features: ConfigOr[seqio.preprocessors.OutputFeaturesType], |
| 136 | copy_pretokenized: bool = True, |
| 137 | with_eos: bool = False, |
| 138 | ) -> input_tf_data.DatasetToDatasetFn: |
| 139 | """Tokenizes output_features using SeqIO. |
| 140 | |
| 141 | Args: |
| 142 | output_features: A mapping from field name to seqio.preprocessors.OutputFeaturesType. |
| 143 | Their vocabulary attribute will be used to tokenize the specified feature. |
| 144 | Alternatively, a config which instantiates to one. |
| 145 | copy_pretokenized: Whether to pass through copies of original features |
| 146 | with "_pretokenized" suffix added to the key. |
| 147 | with_eos: Whether to append EOS to the end of the sequence. |
| 148 | |
| 149 | Returns: |
| 150 | A DatasetToDatasetFn which tokenizes the fields specified in output_features. |
| 151 | """ |
| 152 | output_features = maybe_instantiate(output_features) |
| 153 | return functools.partial( |
| 154 | seqio.preprocessors.tokenize, |
| 155 | output_features=output_features, |
| 156 | copy_pretokenized=copy_pretokenized, |
| 157 | with_eos=with_eos, |
| 158 | ) |
| 159 | |
| 160 | |
| 161 | def add_token_type_ids( |