| 162 | ) |
| 163 | |
| 164 | def stream_generate( |
| 165 | self, |
| 166 | prompt: str, |
| 167 | *, |
| 168 | system: Optional[str] = None, |
| 169 | temperature: float = 0.0, |
| 170 | max_tokens: int = 1024, |
| 171 | ) -> Iterator[str]: |
| 172 | messages: list[dict[str, str]] = [] |
| 173 | if system: |
| 174 | messages.append({"role": "system", "content": system}) |
| 175 | messages.append({"role": "user", "content": prompt}) |
| 176 | |
| 177 | stream = self._client.chat.completions.create( |
| 178 | model=self.model, |
| 179 | messages=messages, |
| 180 | temperature=temperature, |
| 181 | max_tokens=max_tokens, |
| 182 | stream=True, |
| 183 | ) |
| 184 | for chunk in stream: |
| 185 | choices = _field(chunk, "choices", []) or [] |
| 186 | if not choices: |
| 187 | continue |
| 188 | delta = _field(choices[0], "delta", None) |
| 189 | text = _field(delta, "content", None) if delta is not None else None |
| 190 | if text: |
| 191 | yield str(text) |
| 192 | |
| 193 | |
| 194 | class AnthropicLLM: |