Function create_response

litellm/proxy/common_request_processing.py:610–755

Create streaming response, checking if the first chunk is an error. If the first chunk is an error, return a standard JSON error response. Otherwise, return StreamingResponse and stream all content.

(
    generator: AsyncGenerator[str, None],
    media_type: str,
    headers: dict,
    default_status_code: int = status.HTTP_200_OK,
    request: Request | None = None,
)

async def create_response(
    generator: AsyncGenerator[str, None],
    media_type: str,
    headers: dict,
    default_status_code: int = status.HTTP_200_OK,
    request: Request | None = None,
) -> StreamingResponse | JSONResponse:
    """
    Create streaming response, checking if the first chunk is an error.
    If the first chunk is an error, return a standard JSON error response.
    Otherwise, return StreamingResponse and stream all content.
    """
    # Tell buffering reverse proxies (nginx, ingress-nginx, Envoy) to flush SSE
    # immediately instead of releasing the whole stream in one batch (issue #28384).
    streaming_headers: Final = {
        **headers,
        "Cache-Control": "no-cache",
        "X-Accel-Buffering": "no",
    }
    first_chunk_value: str | None = None
    final_status_code = default_status_code

    try:
        # Handle coroutine that returns a generator
        if asyncio.iscoroutine(generator):
            generator = await generator

        # Now get the first chunk from the actual generator
        first_chunk_value = await _buffer_first_chunk_honoring_disconnect(generator, request)

        if first_chunk_value is not None:
            try:
                error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value)
                if error_code_from_chunk is not None:
                    # First chunk is an error, stream hasn't really started yet
                    # Should return standard JSON error response instead of SSE format
                    final_status_code = error_code_from_chunk
                    verbose_proxy_logger.debug(
                        "Error detected in first stream chunk. Returning JSON error response with status code: %s",
                        final_status_code,
                    )

                    # Parse error content
                    error_dict: Final = _extract_error_from_sse_chunk(first_chunk_value)

                    # Consume and close generator (avoid resource leak)
                    try:
                        await generator.aclose()
                    except Exception:
                        pass

                    # Return JSON format error response
                    return JSONResponse(
                        status_code=final_status_code,
                        content={"error": error_dict},
                        headers=headers,
                    )
            except Exception as e: