Initializes the engine and starts its sub-services. If `api_server_pid` is defined, will launch a thread to keep getting request from zmq_server. NOTE: To clarify the launch order of the components of the LLM engine: 1. First, launch splitwise scheduler (if
(self, api_server_pid=None)
| 103 | tracing.trace_set_thread_info("engine") |
| 104 | |
| 105 | def start(self, api_server_pid=None): |
| 106 | """ |
| 107 | Initializes the engine and starts its sub-services. |
| 108 | If `api_server_pid` is defined, will launch a thread |
| 109 | to keep getting request from zmq_server. |
| 110 | |
| 111 | NOTE: To clarify the launch order of the components of the LLM engine: |
| 112 | 1. First, launch splitwise scheduler (if necessary) and expert services (if necessary). |
| 113 | 2. Then, launch common engine, which includes some background threads that inserts tasks and receives ouptuts. |
| 114 | 3. Most importantly, launch workers and cache services. The launch order of them are listed as follows. |
| 115 | |
| 116 | | Profile | Mixed | PrefixCache | Cache -> Worker | Worker -> Cache | |
| 117 | |---------|-------|-------------|-----------------|-----------------| |
| 118 | | 1 | 1 | 1 | 0 | 1 | |
| 119 | | 1 | 1 | 0 | 0 | 0 | |
| 120 | | 1 | 0 | 1 | 0 | 1 | |
| 121 | | 1 | 0 | 0 | 0 | 1 | |
| 122 | | 0 | 1 | 1 | 0 | 1 | |
| 123 | | 0 | 1 | 0 | 0 | 0 | |
| 124 | | 0 | 0 | 1 | 1 | 0 | |
| 125 | | 0 | 0 | 0 | 1 | 0 | |
| 126 | |
| 127 | 4. Finally, inform user the engine has successfully started. |
| 128 | |
| 129 | """ |
| 130 | assert not self.is_started, "The engine is already started." |
| 131 | start_time = time.time() |
| 132 | |
| 133 | self.api_server_pid = api_server_pid |
| 134 | self.ipc_signal_suffix = self.cfg.parallel_config.engine_worker_queue_port[0] |
| 135 | self._init_worker_signals() |
| 136 | |
| 137 | self.launch_components() |
| 138 | |
| 139 | self.engine.start() |
| 140 | self.engine.create_data_processor() |
| 141 | self.data_processor = self.engine.data_processor |
| 142 | |
| 143 | # If block numer is specified and model is deployed in mixed mode, start cache manager first |
| 144 | if not self.do_profile and self.cfg.scheduler_config.splitwise_role != "mixed": |
| 145 | if not current_platform.is_intel_hpu(): |
| 146 | device_ids = self.cfg.parallel_config.device_ids.split(",") |
| 147 | self.cache_manager_processes = self.engine.start_cache_service(device_ids, self.ipc_signal_suffix) |
| 148 | |
| 149 | # Start workers |
| 150 | self.worker_proc = self._start_worker_service() |
| 151 | console_logger.info("Waiting for worker processes to be ready...") |
| 152 | time.sleep(5) |
| 153 | self.worker_init_status = dict() |
| 154 | |
| 155 | result_container = {} |
| 156 | |
| 157 | def check_worker_initialize_status_func(res: dict): |
| 158 | res["worker_is_alive"] = True |
| 159 | if not self.check_worker_initialize_status(): |
| 160 | console_logger.error("Failed to launch worker processes, check log/workerlog.* for more details.") |
| 161 | res["worker_is_alive"] = False |
| 162 |
no test coverage detected