| 6 | |
| 7 | |
| 8 | class T5Model(LLM): |
| 9 | model_name: str = "" |
| 10 | tokenizer: T5Tokenizer = None |
| 11 | model: T5ForConditionalGeneration = None |
| 12 | |
| 13 | def __init__(self, huggingface_model_name: str) -> None: |
| 14 | super().__init__() |
| 15 | self.model_name = huggingface_model_name |
| 16 | self.tokenizer = T5Tokenizer.from_pretrained(self.model_name) |
| 17 | self.model = T5ForConditionalGeneration.from_pretrained(self.model_name) |
| 18 | |
| 19 | @property |
| 20 | def _llm_type(self) -> str: |
| 21 | return self.model_name |
| 22 | |
| 23 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: |
| 24 | |
| 25 | inputs = self.tokenizer( |
| 26 | prompt, |
| 27 | padding=True, |
| 28 | max_length=self.tokenizer.model_max_length, |
| 29 | truncation=True, |
| 30 | return_tensors="pt" |
| 31 | ) |
| 32 | |
| 33 | # inputs_len = inputs["input_ids"].shape[1] |
| 34 | |
| 35 | generated_outputs = self.model.generate( |
| 36 | inputs["input_ids"], |
| 37 | max_new_tokens=512, |
| 38 | ) |
| 39 | decoded_output = self.tokenizer.batch_decode( |
| 40 | generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False) |
| 41 | |
| 42 | output = decoded_output[0] |
| 43 | return output |
| 44 | |
| 45 | |
| 46 | @property |
| 47 | def _identifying_params(self) -> Mapping[str, Any]: |
| 48 | """Get the identifying parameters.""" |
| 49 | return {"model_name": self.model_name} |
| 50 | |
| 51 | if __name__ == "__main__": |
| 52 | llm = T5Model("t5-small") |