return: inputs, tokenizer_kwargs
(self, example: Dict[str, Any], **kwargs)
| 430 | return res if not streaming else inputs |
| 431 | |
| 432 | def _encode(self, example: Dict[str, Any], **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]: |
| 433 | """return: inputs, tokenizer_kwargs""" |
| 434 | messages = example['messages'] |
| 435 | is_multi_modal: bool = any([example.get(key) for key in Template.special_keys]) |
| 436 | |
| 437 | inputs, tokenizer_kwargs = self._concat_and_tokenize( |
| 438 | messages, |
| 439 | self.truncation_strategy, |
| 440 | auto_add_bos=self.auto_add_bos, |
| 441 | is_multi_modal=is_multi_modal, |
| 442 | example=example) |
| 443 | if inputs.get('labels') is None: |
| 444 | inputs.pop('loss_scale', None) |
| 445 | return inputs, tokenizer_kwargs |
| 446 | |
| 447 | def _concat_context_list( |
| 448 | self, |
no test coverage detected