Convert multi-turn messages into ID sequences. Args: messages (List[List[Dict[str, Any]]]): multi-turn messages. Returns: List[int]: ID sequences
(self, request, **kwargs)
| 593 | return tokens["input_ids"][0] |
| 594 | |
| 595 | def messages2ids(self, request, **kwargs): |
| 596 | """ |
| 597 | Convert multi-turn messages into ID sequences. |
| 598 | |
| 599 | Args: |
| 600 | messages (List[List[Dict[str, Any]]]): multi-turn messages. |
| 601 | |
| 602 | Returns: |
| 603 | List[int]: ID sequences |
| 604 | """ |
| 605 | |
| 606 | if "add_generation_prompt" not in kwargs: |
| 607 | kwargs["add_generation_prompt"] = request.get("add_generation_prompt", True) |
| 608 | |
| 609 | spliced_message = self.tokenizer.apply_chat_template( |
| 610 | request, |
| 611 | tokenize=False, |
| 612 | split_special_tokens=False, |
| 613 | add_special_tokens=False, |
| 614 | **kwargs, |
| 615 | ) |
| 616 | request["prompt_tokens"] = spliced_message |
| 617 | req_id = None |
| 618 | tokens = self.tokenizer.tokenize(spliced_message) |
| 619 | if isinstance(request, dict): |
| 620 | req_id = request.get("request_id", None) |
| 621 | token_ids = self.tokenizer.convert_tokens_to_ids(tokens) |
| 622 | data_processor_logger.info(f"req_id:{req_id}, tokens:{tokens}, token_ids: {token_ids}") |
| 623 | return token_ids |
| 624 | |
| 625 | def ids2tokens(self, token_id, task_id): |
| 626 | """ |