gRPC servicer for the tinygrad backend.
| 160 | # --------------------------------------------------------------------------- |
| 161 | |
| 162 | class BackendServicer(backend_pb2_grpc.BackendServicer): |
| 163 | """gRPC servicer for the tinygrad backend.""" |
| 164 | |
| 165 | def __init__(self) -> None: |
| 166 | self._reset_state() |
| 167 | |
| 168 | def _reset_state(self) -> None: |
| 169 | self.model_ref: Optional[str] = None |
| 170 | self.model_type: str = "llm" |
| 171 | self.options: dict[str, str] = {} |
| 172 | # LLM state |
| 173 | self.llm_model = None |
| 174 | self.llm_config: dict = {} |
| 175 | self.llm_tokenizer = None |
| 176 | self.llm_eos_ids: list[int] = [] |
| 177 | self.chat_template: Optional[str] = None |
| 178 | self.tool_parser = resolve_parser(None) |
| 179 | self.max_context = 4096 |
| 180 | # Stable Diffusion state |
| 181 | self.sd_model = None |
| 182 | # Whisper state |
| 183 | self.whisper_model = None |
| 184 | self.whisper_tokenizer = None |
| 185 | |
| 186 | # --------------------- helpers -------------------------------------- |
| 187 | |
| 188 | @staticmethod |
| 189 | def _parse_options(options_list) -> dict[str, str]: |
| 190 | opts: dict[str, str] = {} |
| 191 | for opt in options_list: |
| 192 | if ":" not in opt: |
| 193 | continue |
| 194 | key, value = opt.split(":", 1) |
| 195 | opts[key.strip()] = value.strip() |
| 196 | return opts |
| 197 | |
| 198 | @staticmethod |
| 199 | def _detect_model_type(model_ref: str, explicit: Optional[str]) -> str: |
| 200 | if explicit: |
| 201 | return explicit |
| 202 | name = (model_ref or "").lower() |
| 203 | if "whisper" in name: |
| 204 | return "whisper" |
| 205 | if "sdxl" in name: |
| 206 | return "sdxl" |
| 207 | if "sd-v1" in name or "v1-5" in name or "stable-diffusion" in name: |
| 208 | return "sd15" |
| 209 | if any(tag in name for tag in ("bge", "e5", "minilm", "bert")): |
| 210 | return "bert" |
| 211 | return "llm" |
| 212 | |
| 213 | def _messages_to_dicts(self, messages) -> list[dict]: |
| 214 | result = [] |
| 215 | for msg in messages: |
| 216 | d: dict = {"role": msg.role, "content": msg.content or ""} |
| 217 | if msg.name: |
| 218 | d["name"] = msg.name |
| 219 | if msg.tool_call_id: |