Method streaming_response

python/src/llm/client.rs:1245–1310

Stream and print the response in real-time Returns the complete response as a string after streaming Usage in Python: response = await chunks.streaming_response() # Chunks are printed in real-time, then full response is returned

(&'a self, py: Python<'a>)

    ///     response = await chunks.streaming_response()
    ///     # Chunks are printed in real-time, then full response is returned
    fn streaming_response<'a>(&'a self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
        let provider = Arc::clone(&self.provider);
        let prompt = self.prompt.clone();
        let max_tokens = self.max_tokens;
        let temperature = self.temperature;
        let stream = Arc::clone(&self.stream);
        let initialized = Arc::clone(&self.initialized);

        pyo3_async_runtimes::tokio::future_into_py(py, async move {
            // Phase 1: Initialize stream if needed (using OnceCell - no lock held during init)
            let init_result = initialized
                .get_or_try_init(|| async {
                    // Build request
                    let mut request = LlmRequest::new(prompt.clone());
                    if let Some(tokens) = max_tokens {
                        request = request.with_max_tokens(tokens);
                    }
                    if let Some(temp) = temperature {
                        request = request.with_temperature(temp);
                    }

                    // Get stream from provider (NO MUTEX HELD during this await!)
                    let guard = provider.read().await;
                    let provider_stream = guard.stream(request).await.map_err(to_py_error)?;
                    drop(guard); // Explicitly drop read lock before acquiring mutex

                    // Now briefly hold mutex just to store the stream
                    let mut stream_guard = stream.lock().await;
                    *stream_guard = Some(Box::pin(provider_stream));
                    drop(stream_guard); // Explicitly release mutex

                    Ok::<(), PyErr>(())
                })
                .await;

            // Check if initialization failed
            init_result?;

            // Phase 2: Collect and print chunks in real-time
            let mut full_response = String::new();

            // Hold mutex for streaming - but now we're just polling, not doing heavy I/O
            let mut stream_guard = stream.lock().await;
            if let Some(ref mut s) = *stream_guard {
                while let Some(result) = s.next().await {
                    match result {
                        Ok(response) => {
                            // Print chunk immediately (to stdout) - handle errors gracefully
                            print!("{}", response.content);
                            use std::io::Write;
                            let _ = std::io::stdout().flush(); // Don't panic if flush fails

                            // Accumulate for final response
                            full_response.push_str(&response.content);
                        }
                        Err(e) => {
                            // Always propagate errors
                            return Err(to_py_error(e));