Function generate

trainer/src/inference_server/main.py:156–159

(request: Input)

@stub.function(timeout=1 * 60 * 60, allow_concurrent_inputs=20)
@modal.web_endpoint(method="POST", label=APP_NAME)
async def generate(request: Input) -> Output:
    logging.info(f"Generating for model {request.model}")
    model = Model(request.model)
    return await model.generate.remote.aio(request)