load engine
()
| 117 | |
| 118 | |
| 119 | def load_engine(): |
| 120 | """ |
| 121 | load engine |
| 122 | """ |
| 123 | global llm_engine |
| 124 | if llm_engine is not None: |
| 125 | return llm_engine |
| 126 | |
| 127 | api_server_logger.info(f"FastDeploy LLM API server starting... {os.getpid()}, port: {args.port}") |
| 128 | engine_args = EngineArgs.from_cli_args(args) |
| 129 | if envs.FD_ENABLE_ASYNC_LLM: |
| 130 | engine = AsyncLLM.from_engine_args(engine_args, pid=args.port) |
| 131 | else: |
| 132 | engine = LLMEngine.from_engine_args(engine_args) |
| 133 | started = False |
| 134 | if isinstance(engine, AsyncLLM): |
| 135 | started = asyncio.run(engine.start()) |
| 136 | else: |
| 137 | started = engine.start(api_server_pid=args.port) |
| 138 | if not started: |
| 139 | api_server_logger.error( |
| 140 | "Failed to initialize FastDeploy LLM engine, service exit now!" |
| 141 | "Please check the log file for more details." |
| 142 | ) |
| 143 | return None |
| 144 | llm_engine = engine |
| 145 | return engine |
| 146 | |
| 147 | |
| 148 | def load_data_service(): |
no test coverage detected