Ray executor.
| 231 | |
| 232 | |
| 233 | class RayExecutor(ExecutorBase): |
| 234 | """Ray executor.""" |
| 235 | |
| 236 | def __init__( |
| 237 | self, |
| 238 | model_path: str, |
| 239 | model_config: ModelConfig, |
| 240 | cache_config: CacheConfig, |
| 241 | backend_config: BackendConfig, |
| 242 | dist_config: DistConfig, |
| 243 | misc_config: MiscConfig, |
| 244 | adapters: dict[str, str] = None, |
| 245 | device_type: str = 'cuda', |
| 246 | dtype: str = 'auto', |
| 247 | specdecode_config: SpecDecodeConfig = None, |
| 248 | trust_remote_code: bool = False, |
| 249 | ): |
| 250 | """Initialize Executor.""" |
| 251 | super().__init__( |
| 252 | model_path=model_path, |
| 253 | model_config=model_config, |
| 254 | cache_config=cache_config, |
| 255 | backend_config=backend_config, |
| 256 | dist_config=dist_config, |
| 257 | misc_config=misc_config, |
| 258 | adapters=adapters, |
| 259 | device_type=device_type, |
| 260 | specdecode_config=specdecode_config, |
| 261 | trust_remote_code=trust_remote_code, |
| 262 | ) |
| 263 | |
| 264 | device_ctx = DeviceContext(device_type) |
| 265 | with get_device_manager().context(device_ctx): |
| 266 | logger.info('Init ray cluster.') |
| 267 | attn_tp = dist_config.attn_tp |
| 268 | self.ray_ctx = RayContext(attn_tp, dp=dist_config.dp, device_type=device_type) |
| 269 | placement_group = self.ray_ctx.get_placement_group() |
| 270 | self.placement_group = placement_group |
| 271 | |
| 272 | if self.dp == 1: |
| 273 | self.master_addr = _get_master_addr() |
| 274 | self.master_port = _get_master_port() |
| 275 | else: |
| 276 | self.master_addr = _envs.dp_master_addr |
| 277 | self.master_port = _envs.dp_master_port |
| 278 | if self.master_addr is None or self.master_port is None: |
| 279 | raise RuntimeError('DP > 1 requires "LMDEPLOY_DP_MASTER_ADDR" and "LMDEPLOY_DP_MASTER_PORT".') |
| 280 | |
| 281 | # create workerwrapper actors |
| 282 | worker_kwargs = dict( |
| 283 | model_path=model_path, |
| 284 | cache_config=cache_config, |
| 285 | model_config=model_config, |
| 286 | backend_config=backend_config, |
| 287 | dist_config=dist_config, |
| 288 | misc_config=misc_config, |
| 289 | adapters=adapters, |
| 290 | device_type=device_type, |