(self, message: Message, return_target=False)
| 223 | return tokens |
| 224 | |
| 225 | def encode_message(self, message: Message, return_target=False) -> List[int]: |
| 226 | tokens, targets = [], [] |
| 227 | headers = self.encode_header(message) |
| 228 | contents = self.tokenizer.encode(message["content"].strip(), bos=False, eos=False) |
| 229 | contents.append(self.tokenizer.special_tokens["<|eot_id|>"]) |
| 230 | tokens = headers + contents |
| 231 | |
| 232 | if message["role"] == "assistant": |
| 233 | targets = [-1] * len(headers) + contents |
| 234 | else: |
| 235 | targets = [-1] * len(tokens) |
| 236 | |
| 237 | if return_target: |
| 238 | return tokens, targets |
| 239 | |
| 240 | return tokens, None |
| 241 | |
| 242 | def encode_dialog_prompt(self, dialog: Dialog, completion=False, return_target=False) -> List[int]: |
| 243 | tokens = [self.tokenizer.special_tokens["<|begin_of_text|>"]] |
no test coverage detected