Method
detokenize
(
self,
tokens: List[int],
prev_tokens: Optional[List[int]] = None,
special: bool = False,
)
Source from the content-addressed store, hash-verified
| 52 | return self._model.tokenize(text, add_bos=add_bos, special=special) |
| 53 | |
| 54 | def detokenize( |
| 55 | self, |
| 56 | tokens: List[int], |
| 57 | prev_tokens: Optional[List[int]] = None, |
| 58 | special: bool = False, |
| 59 | ) -> bytes: |
| 60 | return self._model.detokenize(tokens, special=special) |
| 61 | |
| 62 | def encode( |
| 63 | self, text: str, add_bos: bool = True, special: bool = True |
Tested by
no test coverage detected