(self, s)
| 240 | self._decode_token_separator = decode_token_separator |
| 241 | |
| 242 | def encode(self, s): |
| 243 | s = tf.compat.as_text(s) |
| 244 | if self.lowercase: |
| 245 | s = s.lower() |
| 246 | ids = [] |
| 247 | |
| 248 | for token in self._tokenizer.tokenize(s): |
| 249 | int_id = self._token_to_id.get(token, -1) |
| 250 | if int_id < 0: |
| 251 | int_id = self._oov_bucket(token) |
| 252 | if int_id is None: |
| 253 | raise ValueError('Out of vocabulary token %s' % token) |
| 254 | ids.append(int_id) |
| 255 | |
| 256 | # This increments the ids of all words in ids by one, to ensure that 0 |
| 257 | # is preserved as the padding token |
| 258 | return tfds.deprecated.text.text_encoder.pad_incr(ids) |
| 259 | |
| 260 | def decode(self, ids): |
| 261 | # This decrements the ids of all words in ids by one, to ensure that 0 |
no test coverage detected