generate stream api
(request: dict)
| 58 | |
| 59 | @app.post("/generate") |
| 60 | async def generate(request: dict): |
| 61 | """ |
| 62 | generate stream api |
| 63 | """ |
| 64 | api_server_logger.info(f"Receive request: {request}") |
| 65 | stream = request.get("stream", 0) |
| 66 | |
| 67 | if not stream: |
| 68 | output = {} |
| 69 | try: |
| 70 | # 将生成过程包裹在try块中以捕获异常 |
| 71 | for result in llm_engine.generate(request, stream): |
| 72 | output = result |
| 73 | except Exception as e: |
| 74 | # 记录完整的异常堆栈信息 |
| 75 | api_server_logger.error(f"Error during generation: {e!s}", exc_info=True) |
| 76 | # 返回结构化的错误消息并终止流 |
| 77 | output = {"error": str(e), "error_type": e.__class__.__name__} |
| 78 | return output |
| 79 | |
| 80 | async def event_generator(): |
| 81 | try: |
| 82 | # 将生成过程包裹在try块中以捕获异常 |
| 83 | for result in llm_engine.generate(request, stream): |
| 84 | yield f"data: {json.dumps(result)}\n\n" |
| 85 | except Exception as e: |
| 86 | # 记录完整的异常堆栈信息 |
| 87 | api_server_logger.error(f"Error during generation: {e!s}", exc_info=True) |
| 88 | # 返回结构化的错误消息并终止流 |
| 89 | error_msg = {"error": str(e), "error_type": e.__class__.__name__} |
| 90 | yield f"data: {json.dumps(error_msg)}\n\n" |
| 91 | |
| 92 | return StreamingResponse(event_generator(), media_type="text/event-stream") |
| 93 | |
| 94 | |
| 95 | def launch_api_server(args) -> None: |
nothing calls this directly
no test coverage detected