| 285 | } |
| 286 | |
| 287 | async fn stream( |
| 288 | &self, |
| 289 | request: LlmRequest, |
| 290 | ) -> GraphBitResult<Box<dyn Stream<Item = GraphBitResult<LlmResponse>> + Unpin + Send>> { |
| 291 | let url = format!("{}/chat/completions", self.base_url); |
| 292 | |
| 293 | let messages: Vec<MistralAiMessage> = |
| 294 | request.messages.iter().map(Self::convert_message).collect(); |
| 295 | |
| 296 | let tools: Option<Vec<MistralAiTool>> = if request.tools.is_empty() { |
| 297 | None |
| 298 | } else { |
| 299 | Some(request.tools.iter().map(Self::convert_tool).collect()) |
| 300 | }; |
| 301 | |
| 302 | let body = MistralAiStreamRequest { |
| 303 | model: self.model.clone(), |
| 304 | messages, |
| 305 | max_tokens: request.max_tokens, |
| 306 | temperature: request.temperature, |
| 307 | top_p: request.top_p, |
| 308 | tools: tools.clone(), |
| 309 | tool_choice: if tools.is_some() { |
| 310 | Some("auto".to_string()) |
| 311 | } else { |
| 312 | None |
| 313 | }, |
| 314 | stream: Some(true), |
| 315 | }; |
| 316 | |
| 317 | // Add extra parameters |
| 318 | let mut request_json = serde_json::to_value(&body)?; |
| 319 | if let serde_json::Value::Object(ref mut map) = request_json { |
| 320 | for (key, value) in request.extra_params { |
| 321 | map.insert(key, value); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | // Timeout constants for different phases of the request |
| 326 | const CONNECTION_TIMEOUT: Duration = Duration::from_secs(60); |
| 327 | const ERROR_BODY_TIMEOUT: Duration = Duration::from_secs(10); |
| 328 | const CHUNK_TIMEOUT: Duration = Duration::from_secs(30); |
| 329 | |
| 330 | // Apply timeout to initial connection |
| 331 | let response = timeout( |
| 332 | CONNECTION_TIMEOUT, |
| 333 | self.client |
| 334 | .post(&url) |
| 335 | .header("Authorization", format!("Bearer {}", self.api_key)) |
| 336 | .header("Content-Type", "application/json") |
| 337 | .json(&request_json) |
| 338 | .send(), |
| 339 | ) |
| 340 | .await |
| 341 | .map_err(|_| { |
| 342 | GraphBitError::llm_provider( |
| 343 | "mistralai", |
| 344 | format!( |