| 166 | # Tokenization |
| 167 | |
| 168 | def tokenize(self, text: bytes, add_bos: bool, special: bool): |
| 169 | n_ctx = self.n_ctx_train() |
| 170 | tokens = (llama_cpp.llama_token * n_ctx)() |
| 171 | n_tokens = llama_cpp.llama_tokenize( |
| 172 | self.vocab, text, len(text), tokens, n_ctx, add_bos, special |
| 173 | ) |
| 174 | if n_tokens < 0: |
| 175 | n_tokens = abs(n_tokens) |
| 176 | tokens = (llama_cpp.llama_token * n_tokens)() |
| 177 | n_tokens = llama_cpp.llama_tokenize( |
| 178 | self.vocab, text, len(text), tokens, n_tokens, add_bos, special |
| 179 | ) |
| 180 | if n_tokens < 0: |
| 181 | raise RuntimeError( |
| 182 | f'Failed to tokenize: text="{text}" n_tokens={n_tokens}' |
| 183 | ) |
| 184 | return list(tokens[:n_tokens]) |
| 185 | |
| 186 | def token_to_piece(self, token: int, special: bool = False) -> bytes: |
| 187 | buf = ctypes.create_string_buffer(32) |