(self,
path: str,
max_seq_len: int = 2048,
tokenizer_only: bool = False,
tokenizer_path: Optional[str] = None,
model_config: Optional[str] = None,
tokenizer_type: Optional[str] = 'v7',
meta_template: Optional[Dict] = None)
| 9 | class InternLM(BaseModel): |
| 10 | |
| 11 | def __init__(self, |
| 12 | path: str, |
| 13 | max_seq_len: int = 2048, |
| 14 | tokenizer_only: bool = False, |
| 15 | tokenizer_path: Optional[str] = None, |
| 16 | model_config: Optional[str] = None, |
| 17 | tokenizer_type: Optional[str] = 'v7', |
| 18 | meta_template: Optional[Dict] = None): |
| 19 | if tokenizer_only: |
| 20 | self._load_tokenizer(tokenizer_path=tokenizer_path, |
| 21 | tokenizer_type=tokenizer_type, |
| 22 | max_seq_len=max_seq_len) |
| 23 | else: |
| 24 | self._load_model(path=path, |
| 25 | max_seq_len=max_seq_len, |
| 26 | tokenizer_path=tokenizer_path, |
| 27 | tokenizer_type=tokenizer_type, |
| 28 | model_config=model_config) |
| 29 | self.template_parser = LMTemplateParser(meta_template) |
| 30 | self.eos_token_id = None |
| 31 | if meta_template and 'eos_token_id' in meta_template: |
| 32 | self.eos_token_id = meta_template['eos_token_id'] |
| 33 | |
| 34 | def _load_model(self, |
| 35 | path: str, |
nothing calls this directly
no test coverage detected