Generate completion for the request. The request should be a JSON object with the following fields: - prompt: the prompt to use for the generation. - stream: whether to stream the results or not. - other fields: the sampling parameters (See `SamplingParams` for details).
(request: Request)
| 24 | |
| 25 | @app.post("/generate") |
| 26 | async def generate(request: Request) -> Response: |
| 27 | """Generate completion for the request. |
| 28 | |
| 29 | The request should be a JSON object with the following fields: |
| 30 | - prompt: the prompt to use for the generation. |
| 31 | - stream: whether to stream the results or not. |
| 32 | - other fields: the sampling parameters (See `SamplingParams` for details). |
| 33 | """ |
| 34 | request_dict = await request.json() |
| 35 | prompt = request_dict.pop("prompt") |
| 36 | stream = request_dict.pop("stream", False) |
| 37 | sampling_params = SamplingParams(**request_dict) |
| 38 | request_id = random_uuid() |
| 39 | |
| 40 | results_generator = engine.generate(prompt, sampling_params, request_id) |
| 41 | |
| 42 | # Streaming case |
| 43 | async def stream_results() -> AsyncGenerator[bytes, None]: |
| 44 | async for request_output in results_generator: |
| 45 | prompt = request_output.prompt |
| 46 | text_outputs = [ |
| 47 | prompt + output.text for output in request_output.outputs |
| 48 | ] |
| 49 | ret = {"text": text_outputs} |
| 50 | yield (json.dumps(ret) + "\0").encode("utf-8") |
| 51 | |
| 52 | if stream: |
| 53 | return StreamingResponse(stream_results()) |
| 54 | |
| 55 | # Non-streaming case |
| 56 | final_output = None |
| 57 | async for request_output in results_generator: |
| 58 | if await request.is_disconnected(): |
| 59 | # Abort the request if the client disconnects. |
| 60 | await engine.abort(request_id) |
| 61 | return Response(status_code=499) |
| 62 | final_output = request_output |
| 63 | |
| 64 | assert final_output is not None |
| 65 | prompt = final_output.prompt |
| 66 | text_outputs = [prompt + output.text for output in final_output.outputs] |
| 67 | ret = {"text": text_outputs} |
| 68 | return JSONResponse(ret) |
| 69 | |
| 70 | |
| 71 | if __name__ == "__main__": |
nothing calls this directly
no test coverage detected