De-tokenize. Args: t: a list of token ids. offset: for incrementally decoding. Default to None, which means not applied. skip_special_tokens: Whether or not to remove special tokens in the decoding. Returns:
(self, t: Sequence[int], offset: int | None = None, skip_special_tokens: bool = True)
| 209 | return encoded |
| 210 | |
| 211 | def decode(self, t: Sequence[int], offset: int | None = None, skip_special_tokens: bool = True): |
| 212 | """De-tokenize. |
| 213 | |
| 214 | Args: |
| 215 | t: a list of token ids. |
| 216 | offset: for incrementally decoding. Default to None, which |
| 217 | means not applied. |
| 218 | skip_special_tokens: Whether or not to remove special |
| 219 | tokens in the decoding. |
| 220 | |
| 221 | Returns: |
| 222 | str: text of decoding tokens. |
| 223 | """ |
| 224 | t = t[offset:] |
| 225 | out_string = self.model.decode(t, skip_special_tokens=skip_special_tokens) |
| 226 | if offset: |
| 227 | logger = get_logger('lmdeploy') |
| 228 | logger.warning('For incrementally detokenization, please try ' |
| 229 | 'detokenize_incrementally function instead.') |
| 230 | out_string = self._maybe_add_prefix_space(t, out_string) |
| 231 | return out_string |
| 232 | |
| 233 | @staticmethod |
| 234 | def _convert_tokens_to_string_with_added_encoders( |
no test coverage detected