Function get_event_publisher

llama_cpp/server/app.py:191–222

(
    request: Request,
    inner_send_chan: MemoryObjectSendStream[typing.Any],
    body: CreateCompletionRequest | CreateChatCompletionRequest,
    body_model: str | None,
    llama_call,
    kwargs,
)

async def get_event_publisher(
    request: Request,
    inner_send_chan: MemoryObjectSendStream[typing.Any],
    body: CreateCompletionRequest | CreateChatCompletionRequest,
    body_model: str | None,
    llama_call,
    kwargs,
):
    server_settings = next(get_server_settings())
    interrupt_requests = (
        server_settings.interrupt_requests if server_settings else False
    )
    async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy:
        llama = prepare_request_resources(body, llama_proxy, body_model, kwargs)
        async with inner_send_chan:
            try:
                iterator = await run_in_threadpool(llama_call, llama, **kwargs)
                async for chunk in iterate_in_threadpool(iterator):
                    await inner_send_chan.send(dict(data=json.dumps(chunk)))
                    if await request.is_disconnected():
                        raise anyio.get_cancelled_exc_class()()
                    if interrupt_requests and llama_outer_lock.locked():
                        await inner_send_chan.send(dict(data="[DONE]"))
                        raise anyio.get_cancelled_exc_class()()
                await inner_send_chan.send(dict(data="[DONE]"))
            except anyio.get_cancelled_exc_class() as e:
                print("disconnected")
                with anyio.move_on_after(1, shield=True):
                    print(
                        f"Disconnected from client (via refresh/close) {request.client}"
                    )
                    raise e


def _logit_bias_tokens_to_input_ids(