(self, message: Message)
| 207 | return decoded_str |
| 208 | |
| 209 | def encode_header(self, message: Message) -> List[int]: |
| 210 | tokens = [] |
| 211 | if message["role"] == "system": |
| 212 | tokens.extend(self.tokenizer.encode("System: ", bos=False, eos=False)) |
| 213 | elif message["role"] == "user": |
| 214 | tokens.extend(self.tokenizer.encode("User: ", bos=False, eos=False)) |
| 215 | elif message["role"] == "assistant": |
| 216 | tokens.extend(self.tokenizer.encode("Assistant: ", bos=False, eos=False)) |
| 217 | else: |
| 218 | raise NotImplementedError(f"Role {message['role']} not implemented.") |
| 219 | # tokens.append(self.tokenizer.special_tokens["<|start_header_id|>"]) |
| 220 | # tokens.extend(self.tokenizer.encode(message["role"], bos=False, eos=False)) |
| 221 | # tokens.append(self.tokenizer.special_tokens["<|end_header_id|>"]) |
| 222 | # tokens.extend(self.tokenizer.encode("\n\n", bos=False, eos=False)) |
| 223 | return tokens |
| 224 | |
| 225 | def encode_message(self, message: Message, return_target=False) -> List[int]: |
| 226 | tokens, targets = [], [] |
no test coverage detected