(self, dialog: Dialog, completion=False, return_target=False)
| 240 | return tokens, None |
| 241 | |
| 242 | def encode_dialog_prompt(self, dialog: Dialog, completion=False, return_target=False) -> List[int]: |
| 243 | tokens = [self.tokenizer.special_tokens["<|begin_of_text|>"]] |
| 244 | targets = [-1] |
| 245 | for message in dialog: |
| 246 | _tokens, _targets = self.encode_message(message, return_target=return_target) |
| 247 | tokens.extend(_tokens) |
| 248 | if _targets is not None: |
| 249 | targets.extend(_targets) |
| 250 | # Add the start of an assistant message for the model to complete. |
| 251 | if completion: |
| 252 | tokens.extend(self.encode_header({"role": "assistant", "content": ""})) |
| 253 | |
| 254 | if return_target: |
| 255 | return tokens, targets |
| 256 | |
| 257 | return tokens |
no test coverage detected