Function lifespan

fastdeploy/entrypoints/openai/api_server.py:168–301

async context manager for FastAPI lifespan

(app: FastAPI)

@asynccontextmanager
async def lifespan(app: FastAPI):
    """
    async context manager for FastAPI lifespan
    """
    global engine_args
    global _tracing_inited
    import logging

    # Initialize tracing in worker lifecycle instead of module import time.
    # This avoids creating grpc/cygrpc state before gunicorn forks workers.
    if not _tracing_inited:
        tracing.process_tracing_init()
        _tracing_inited = True

    uvicorn_access = logging.getLogger("uvicorn.access")
    uvicorn_access.handlers.clear()

    # 使用 gunicorn 的格式
    formatter = logging.Formatter("[%(asctime)s] [%(process)d] [INFO] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)
    uvicorn_access.addHandler(handler)
    uvicorn_access.propagate = False

    if args.tokenizer is None:
        args.tokenizer = args.model
    pid = args.port
    api_server_logger.info(f"{pid}")

    if args.served_model_name is not None:
        served_model_names = args.served_model_name
        verification = True
    else:
        served_model_names = args.model
        verification = False
    model_paths = [ModelPath(name=served_model_names, model_path=args.model, verification=verification)]

    engine_args = EngineArgs.from_cli_args(args, skip_port_check=True)
    fd_config = engine_args.create_engine_config()
    if envs.FD_ENABLE_ASYNC_LLM:
        os.environ["INFERENCE_MSG_QUEUE_ID"] = engine_args.engine_worker_queue_port[engine_args.local_data_parallel_id]
    engine_client = EngineClient(
        pid=pid,
        port=int(os.environ.get("INFERENCE_MSG_QUEUE_ID", "0")),
        fd_config=fd_config,
        workers=args.workers,
        max_logprobs=args.max_logprobs,
    )
    await engine_client.connection_manager.initialize()
    app.state.dynamic_load_weight = args.dynamic_load_weight
    model_handler = OpenAIServingModels(
        model_paths,
        args.max_model_len,
        args.ips,
    )
    app.state.model_handler = model_handler
    global llm_engine