初始化模型实例。 Args: model_name_or_path (str): 模型名称或路径。 Attributes: model_name_or_path (str): 存储模型名称或路径。 decode_status (dict): 存储解码状态信息。 tokenizer (object): 存储分词器实例。 eos_token_ids (list): 存储结束符号的token ID列表。 eos_token_id_len (int): 存储结束符号的token
| 28 | |
| 29 | |
| 30 | class Ernie4_5Processor(BaseDataProcessor): |
| 31 | """ |
| 32 | 初始化模型实例。 |
| 33 | |
| 34 | Args: |
| 35 | model_name_or_path (str): 模型名称或路径。 |
| 36 | |
| 37 | Attributes: |
| 38 | model_name_or_path (str): 存储模型名称或路径。 |
| 39 | decode_status (dict): 存储解码状态信息。 |
| 40 | tokenizer (object): 存储分词器实例。 |
| 41 | eos_token_ids (list): 存储结束符号的token ID列表。 |
| 42 | eos_token_id_len (int): 存储结束符号的token ID列表的长度。 |
| 43 | pad_token_id (int): 存储填充符号的token ID。 |
| 44 | """ |
| 45 | |
| 46 | def __init__(self, model_name_or_path, reasoning_parser_obj=None, tool_parser_obj=None): |
| 47 | |
| 48 | self.model_name_or_path = model_name_or_path |
| 49 | data_processor_logger.info(f"model_name_or_path: {model_name_or_path}") |
| 50 | |
| 51 | # Generation config |
| 52 | try: |
| 53 | self.generation_config = GenerationConfig.from_pretrained(self.model_name_or_path) |
| 54 | except Exception as e: |
| 55 | data_processor_logger.warning( |
| 56 | f"Can't find generation config, so it will not use " |
| 57 | f"generation_config field in the model config, details={e}" |
| 58 | ) |
| 59 | self.generation_config = None |
| 60 | |
| 61 | self.decode_status = dict() |
| 62 | self.tool_parser_dict = dict() |
| 63 | self.thinking_parser_dict = dict() |
| 64 | self.model_status_dict = dict() |
| 65 | self._load_tokenizer() |
| 66 | data_processor_logger.info( |
| 67 | f"tokenizer information: bos_token is {self.tokenizer.bos_token} \ |
| 68 | {self.tokenizer.bos_token_id}, \ |
| 69 | eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} " |
| 70 | ) |
| 71 | try: |
| 72 | from paddleformers.trl.llm_utils import get_eos_token_id |
| 73 | except Exception: |
| 74 | from paddleformers.cli.utils.llm_utils import get_eos_token_id |
| 75 | |
| 76 | self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config) |
| 77 | self.eos_token_id_len = len(self.eos_token_ids) |
| 78 | self.pad_token_id = self.get_pad_id() |
| 79 | self.reasoning_parser = None |
| 80 | self.tool_parser_obj = tool_parser_obj |
| 81 | if reasoning_parser_obj: |
| 82 | self.reasoning_parser = reasoning_parser_obj(self.tokenizer) |
| 83 | |
| 84 | def process_request(self, request, max_model_len=None, **kwargs): |
| 85 | """ |
| 86 | Preprocess the request |
| 87 |
no outgoing calls