Call LLM with the given prompt. Supports openai-compatible, anthropic, and bedrock providers. For bedrock/anthropic, uses litellm to translate the API call. Args: prompt: The prompt to send config: Configuration containing LLM settings model: Model name (de
(
prompt: str,
config: Config,
model: str = None,
temperature: float = 0.0
)
| 146 | self._cache_registry_key = (cache_registry_key, model_name) |
| 147 | |
| 148 | @property |
| 149 | def _prompt_caching_active(self) -> bool: |
| 150 | return self._prompt_caching_enabled and self._cache_registry_key not in _CACHE_UNSUPPORTED |
| 151 | |
| 152 | async def _map_messages(self, messages, model_request_parameters=None, *, model_settings=None): |
| 153 | base = OpenAIChatModel._map_messages |
| 154 | base_params = set(inspect.signature(base).parameters) |
| 155 | if "model_settings" in base_params: |
| 156 | openai_messages = await super()._map_messages( |
| 157 | messages, model_request_parameters, model_settings=model_settings |
| 158 | ) |
| 159 | elif "model_request_parameters" in base_params: |
| 160 | # Intermediate pydantic-ai versions accept request parameters but |
| 161 | # predate the keyword-only model_settings argument. |
| 162 | openai_messages = await super()._map_messages(messages, model_request_parameters) |
| 163 | else: |
| 164 | # Early pydantic-ai versions accept only messages. |
| 165 | openai_messages = await super()._map_messages(messages) |
| 166 | if self._prompt_caching_active: |
| 167 | # Breakpoint 1: last system/developer message (covers tools + system). |
| 168 | for message in reversed(openai_messages): |
| 169 | if message.get("role") in ("system", "developer"): |
| 170 | _add_cache_control_to_message(message) |
| 171 | break |
| 172 | # Breakpoint 2: final message (covers the whole conversation prefix). |
| 173 | if openai_messages: |
| 174 | _add_cache_control_to_message(openai_messages[-1]) |
| 175 | return openai_messages |
| 176 | |
| 177 | async def _completions_create(self, messages, stream, model_settings, model_request_parameters): |
| 178 | if not self._prompt_caching_active: |
| 179 | return await super()._completions_create( |
| 180 | messages, stream, model_settings, model_request_parameters |
| 181 | ) |
| 182 | try: |
| 183 | return await super()._completions_create( |
| 184 | messages, stream, model_settings, model_request_parameters |
| 185 | ) |
| 186 | except ModelHTTPError as e: |
| 187 | if e.status_code not in (400, 422): |
| 188 | raise |
| 189 | _CACHE_UNSUPPORTED.add(self._cache_registry_key) |
| 190 | logger.warning( |
| 191 | "Provider rejected prompt caching markers for model %s (HTTP %s); " |
| 192 | "retrying without prompt caching.", |
| 193 | self.model_name, |
| 194 | e.status_code, |
| 195 | ) |
| 196 | try: |
| 197 | result = await super()._completions_create( |
| 198 | messages, stream, model_settings, model_request_parameters |
| 199 | ) |
| 200 | except ModelHTTPError: |
| 201 | # The failure wasn't (only) about caching — don't blame the markers. |
| 202 | _CACHE_UNSUPPORTED.discard(self._cache_registry_key) |
| 203 | raise |
| 204 | logger.warning( |
| 205 | "Prompt caching disabled for model %s at this endpoint; " |
no test coverage detected