| 189 | return bytes(buf) |
| 190 | |
| 191 | def detokenize(self, tokens: List[int], special: bool = False) -> bytes: |
| 192 | output = b"" |
| 193 | size = 32 |
| 194 | buffer = (ctypes.c_char * size)() |
| 195 | for token in tokens: |
| 196 | n = llama_cpp.llama_token_to_piece( |
| 197 | self.vocab, llama_cpp.llama_token(token), buffer, size, 0, special |
| 198 | ) |
| 199 | assert n <= size |
| 200 | output += bytes(buffer[:n]) |
| 201 | # NOTE: Llama1 models automatically added a space at the start of the prompt |
| 202 | # this line removes a leading space if the first token is a beginning of sentence token |
| 203 | return ( |
| 204 | output[1:] |
| 205 | if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" " |
| 206 | else output |
| 207 | ) |
| 208 | |
| 209 | # Extra |
| 210 | def metadata(self) -> Dict[str, str]: |