(self, model_name_or_path: str, device: str="cuda", cpu_offloading: bool=False)
| 12 | use_gpu: bool = True |
| 13 | |
| 14 | def __init__(self, model_name_or_path: str, device: str="cuda", cpu_offloading: bool=False) -> None: |
| 15 | super().__init__() |
| 16 | self.model_name = model_name_or_path |
| 17 | self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=False) |
| 18 | self.model = AutoModelForCausalLM.from_pretrained( |
| 19 | model_name_or_path, low_cpu_mem_usage=True |
| 20 | ) |
| 21 | if self.tokenizer.pad_token_id == None: |
| 22 | self.tokenizer.add_special_tokens({"bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>"}) |
| 23 | self.model.resize_token_embeddings(len(self.tokenizer)) |
| 24 | self.use_gpu = (True if device == "cuda" else False) |
| 25 | if (device == "cuda" and not cpu_offloading) or device == "mps": |
| 26 | self.model.to(device) |
| 27 | |
| 28 | @property |
| 29 | def _llm_type(self) -> str: |
nothing calls this directly
no outgoing calls
no test coverage detected