(self, config)
| 45 | |
| 46 | class LocalLLM(LLM): |
| 47 | def __init__(self, config): |
| 48 | super().__init__(config) |
| 49 | openai_api_key = os.environ["TOGETHER_API_KEY"] |
| 50 | openai_api_base = "https://api.together.xyz" |
| 51 | # openai_api_base = "http://localhost:8004/v1" |
| 52 | # openai_api_key="token-abc123" |
| 53 | |
| 54 | |
| 55 | self.client = OpenAI( |
| 56 | api_key=openai_api_key, |
| 57 | base_url=openai_api_base, |
| 58 | ) |
| 59 | |
| 60 | def __call__(self, prompt: str): |
| 61 | prompt = truncate_tokens_hf(prompt, encoding_name=self.config["model"]) |