(self)
| 141 | mtp: MtpConfig = field(default_factory=MtpConfig) |
| 142 | |
| 143 | def __post_init__(self): |
| 144 | import_external_libs(self.external_lib) |
| 145 | |
| 146 | if self.hf_config_path is None: |
| 147 | self.hf_config_path = self.path |
| 148 | if self.tokenizer_path is None: |
| 149 | self.tokenizer_path = self.path |
| 150 | |
| 151 | self.local_path = copy_to_local(self.path, use_shm=self.use_shm) |
| 152 | |
| 153 | # construct tokenizer |
| 154 | if self.load_tokenizer: |
| 155 | self.local_tokenizer_path = copy_to_local(self.tokenizer_path, use_shm=self.use_shm) |
| 156 | self.tokenizer = hf_tokenizer(self.local_tokenizer_path, trust_remote_code=self.trust_remote_code) |
| 157 | self.processor = hf_processor(self.local_tokenizer_path, trust_remote_code=self.trust_remote_code) |
| 158 | |
| 159 | if self.custom_chat_template is not None: |
| 160 | if self.processor is not None: |
| 161 | self.processor.chat_template = self.custom_chat_template |
| 162 | else: |
| 163 | self.tokenizer.chat_template = self.custom_chat_template |
| 164 | |
| 165 | self.local_hf_config_path = copy_to_local(self.hf_config_path, use_shm=self.use_shm) |
| 166 | self.generation_config = get_generation_config( |
| 167 | self.local_hf_config_path, trust_remote_code=self.trust_remote_code |
| 168 | ) |
| 169 | |
| 170 | # construct hf_config |
| 171 | attn_implementation = self.override_config.get("attn_implementation", "flash_attention_2") |
| 172 | self.hf_config = AutoConfig.from_pretrained( |
| 173 | self.local_hf_config_path, trust_remote_code=self.trust_remote_code, attn_implementation=attn_implementation |
| 174 | ) |
| 175 | |
| 176 | override_config_kwargs = {} |
| 177 | |
| 178 | if self.tokenizer is not None: |
| 179 | override_config_kwargs.update( |
| 180 | { |
| 181 | "bos_token_id": self.tokenizer.bos_token_id, |
| 182 | "eos_token_id": self.tokenizer.eos_token_id, |
| 183 | "pad_token_id": self.tokenizer.pad_token_id, |
| 184 | } |
| 185 | ) |
| 186 | |
| 187 | # TODO: (vermouth1992). self.config.model in megatron differs from that of fsdp in the override_config. |
| 188 | override_config = ( |
| 189 | self.override_config["model_config"] if "model_config" in self.override_config else self.override_config |
| 190 | ) |
| 191 | override_config_kwargs.update(override_config) |
| 192 | update_model_config(self.hf_config, override_config_kwargs=override_config_kwargs) |
| 193 | |
| 194 | self.share_embeddings_and_output_weights = getattr(self.hf_config, "tie_word_embeddings", False) |
| 195 | |
| 196 | # get model architectures |
| 197 | self.architectures = getattr(self.hf_config, "architectures", None) |
| 198 | assert self.architectures is not None and len(self.architectures) == 1, ( |
| 199 | "Expect only one architecture, got {}".format(self.architectures) |
| 200 | ) |
nothing calls this directly
no test coverage detected