LLM-based controller for memory metadata generation
| 234 | return json.dumps(empty_response) |
| 235 | |
| 236 | class LLMController: |
| 237 | """LLM-based controller for memory metadata generation""" |
| 238 | def __init__(self, |
| 239 | backend: Literal["openai", "ollama", "sglang"] = "sglang", |
| 240 | model: str = "gpt-4", |
| 241 | api_key: Optional[str] = None, |
| 242 | api_base: Optional[str] = None, |
| 243 | sglang_host: str = "http://localhost", |
| 244 | sglang_port: int = 30000): |
| 245 | if backend == "openai": |
| 246 | self.llm = OpenAIController(model, api_key) |
| 247 | elif backend == "ollama": |
| 248 | # Use LiteLLM to control Ollama with JSON output |
| 249 | ollama_model = f"ollama/{model}" if not model.startswith("ollama/") else model |
| 250 | self.llm = LiteLLMController( |
| 251 | model=ollama_model, |
| 252 | api_base="http://localhost:11434", |
| 253 | api_key="EMPTY" |
| 254 | ) |
| 255 | elif backend == "sglang": |
| 256 | # Direct SGLang API calls (better performance, no proxy) |
| 257 | self.llm = SGLangController(model, sglang_host, sglang_port) |
| 258 | else: |
| 259 | raise ValueError("Backend must be 'openai', 'ollama', or 'sglang'") |
| 260 | |
| 261 | class MemoryNote: |
| 262 | """Basic memory unit with metadata""" |