(
self,
fd_config: FDConfig,
device: str, # logic device
device_id: int, # physical device id
rank: int,
local_rank: int,
)
| 53 | """ """ |
| 54 | |
| 55 | def __init__( |
| 56 | self, |
| 57 | fd_config: FDConfig, |
| 58 | device: str, # logic device |
| 59 | device_id: int, # physical device id |
| 60 | rank: int, |
| 61 | local_rank: int, |
| 62 | ): |
| 63 | super().__init__(fd_config=fd_config, device=device) |
| 64 | self.enable_mm = self.model_config.enable_mm |
| 65 | self.rank = rank |
| 66 | self.local_rank = local_rank |
| 67 | self.device_id = device_id |
| 68 | self.speculative_method = self.fd_config.speculative_config.method |
| 69 | self.speculative_decoding = self.speculative_method is not None |
| 70 | self.enable_logprob = fd_config.model_config.enable_logprob |
| 71 | |
| 72 | self.guided_backend = None |
| 73 | if self.fd_config.structured_outputs_config.guided_decoding_backend != "off": |
| 74 | self.guided_backend = get_guided_backend(fd_config=self.fd_config) |
| 75 | |
| 76 | # Sampler |
| 77 | if not self.speculative_decoding: |
| 78 | self.sampler = Sampler() |
| 79 | else: |
| 80 | self.sampler = SpeculativeSampler(fd_config) |
| 81 | |
| 82 | # Cuda Graph |
| 83 | self.graph_opt_level = self.graph_opt_config.graph_opt_level |
| 84 | self.use_cudagraph = self.graph_opt_config.use_cudagraph |
| 85 | self.cudagraph_capture_sizes = list(reversed(self.graph_opt_config.cudagraph_capture_sizes)) |
| 86 | self.sot_warmup_sizes = self.graph_opt_config.sot_warmup_sizes |
| 87 | |
| 88 | # Initialize share inputs |
| 89 | self._init_share_inputs(self.scheduler_config.max_num_seqs) |
| 90 | self.infer_seed_increment = paddle.full( |
| 91 | shape=[self.scheduler_config.max_num_seqs, 1], |
| 92 | fill_value=4, |
| 93 | dtype="int64", |
| 94 | ).cpu() |
| 95 | self.restore_chunked_prefill_request = dict() |
| 96 | |
| 97 | # Initialize attention Backend |
| 98 | self.attn_backends: list[AttentionBackend] = [] |
| 99 | # self.attn_metadatas: list[AttentionMetadata] = [] |
| 100 | self.initialize_attn_backend() |
| 101 | |
| 102 | # Forward meta store the global meta information of the forward |
| 103 | self.forward_meta: ForwardMeta = None |
| 104 | |
| 105 | # Postprocess Env params |
| 106 | os.environ["INFERENCE_MSG_QUEUE_ID"] = str(self.parallel_config.local_engine_worker_queue_port) |
| 107 | |
| 108 | def exist_prefill(self): |
| 109 | """ |
nothing calls this directly
no test coverage detected