| 42 | self.logger = get_logger() |
| 43 | |
| 44 | def _load_model(self, from_pretrained_kwargs): |
| 45 | from accessory.model.meta import MetaModel |
| 46 | from accessory.util.misc import init_distributed_mode |
| 47 | if not dist.is_initialized(): |
| 48 | init_distributed_mode() |
| 49 | |
| 50 | model_parallel_group = dist.GroupMember.WORLD |
| 51 | from_pretrained_kwargs['mp_group'] = model_parallel_group |
| 52 | |
| 53 | self.model = MetaModel.from_pretrained(**from_pretrained_kwargs) |
| 54 | self.tokenizer = self.model.tokenizer |
| 55 | self.logger = get_logger() |
| 56 | |
| 57 | def _load_tokenizer(self, from_pretrained_kwargs): |
| 58 | from accessory.model.tokenizer import ( |