(
self,
fd_config: FDConfig,
device: str, # logic device
device_id: int, # physical device id
rank: int,
local_rank: int,
)
| 418 | """ """ |
| 419 | |
| 420 | def __init__( |
| 421 | self, |
| 422 | fd_config: FDConfig, |
| 423 | device: str, # logic device |
| 424 | device_id: int, # physical device id |
| 425 | rank: int, |
| 426 | local_rank: int, |
| 427 | ): |
| 428 | super().__init__(fd_config=fd_config, device=device) |
| 429 | self.rank = rank |
| 430 | self.local_rank = local_rank |
| 431 | self.device_id = device_id |
| 432 | self.speculative_method = self.fd_config.speculative_config.method |
| 433 | self.speculative_decoding = self.speculative_method is not None |
| 434 | # This measurement_mode only works in BF16 mode! |
| 435 | self.measurement_mode = True if envs.FD_HPU_MEASUREMENT_MODE == "1" else False |
| 436 | |
| 437 | self.guided_backend = None |
| 438 | if self.fd_config.structured_outputs_config.guided_decoding_backend != "off": |
| 439 | self.guided_backend = get_guided_backend(fd_config=self.fd_config) |
| 440 | |
| 441 | # Sampler |
| 442 | if not self.speculative_decoding: |
| 443 | self.sampler = Sampler(fd_config) |
| 444 | else: |
| 445 | self.sampler = SpeculativeSampler(fd_config) |
| 446 | |
| 447 | # Lazy initialize kv cache after model loading |
| 448 | # self.kv_caches: list[paddle.Tensor] = [] |
| 449 | |
| 450 | # Cuda Graph |
| 451 | self.use_cudagraph = self.graph_opt_config.use_cudagraph |
| 452 | self.cudagraph_capture_sizes = list(reversed(self.graph_opt_config.cudagraph_capture_sizes)) |
| 453 | self.cudagraph_num_of_warmups = self.graph_opt_config.cudagraph_num_of_warmups |
| 454 | self.input_ids = paddle.zeros(self.scheduler_config.max_num_seqs, dtype="int32") |
| 455 | |
| 456 | # Initialize share inputs |
| 457 | self._init_share_inputs(self.scheduler_config.max_num_seqs) |
| 458 | self.infer_seed_increment = paddle.full( |
| 459 | shape=[self.scheduler_config.max_num_seqs, 1], fill_value=4, dtype="int64" |
| 460 | ).cpu() |
| 461 | self.restore_chunked_prefill_request = dict() |
| 462 | |
| 463 | # Initialize attention Backend |
| 464 | # Note(gonshaotian): Currently, all attention layers share one attention backend instance. |
| 465 | # In the future, we will expand it as a list. |
| 466 | self.attn_backends: list[AttentionBackend] = [] |
| 467 | # self.attn_metadatas: list[AttentionMetadata] = [] |
| 468 | self.initialize_attn_backend() |
| 469 | |
| 470 | # Forward meta store the global meta information of the forward |
| 471 | self.forward_meta: HPUForwardMeta = None |
| 472 | self.is_warmuping = False |
| 473 | self.is_hpu_perf_breakdown_sync_mode = int(os.environ.get("HPU_PERF_BREAKDOWN_SYNC_MODE", 1)) == 1 |
| 474 | # Postprocess Env params |
| 475 | os.environ["INFERENCE_MSG_QUEUE_ID"] = str( |
| 476 | self.local_rank + int(self.parallel_config.local_engine_worker_queue_port) |
| 477 | ) |
nothing calls this directly
no test coverage detected