Main process for MTP inference. Args: step_use_cudagraph: bool Whether to use cuda graph. Use the target model flag to avoid hanging problems with EP.
(self, step_use_cudagraph: bool = False, is_dummy_run: bool = False)
| 979 | self.model.empty_input_forward(forward_meta=self.forward_meta) |
| 980 | |
| 981 | def _propose_xpu(self, step_use_cudagraph: bool = False, is_dummy_run: bool = False): |
| 982 | """ |
| 983 | Main process for MTP inference. |
| 984 | Args: |
| 985 | step_use_cudagraph: bool |
| 986 | Whether to use cuda graph. Use the target model flag to avoid hanging problems with EP. |
| 987 | """ |
| 988 | # TODO(chenhuan09)：check multi step |
| 989 | for substep in range(self.num_model_steps): |
| 990 | if self.model_inputs["not_need_stop"]: |
| 991 | self.model_inputs["substep"] = substep |
| 992 | # Remove padding |
| 993 | self.forward_meta = xpu_pre_process( |
| 994 | self.model_inputs["input_ids"], |
| 995 | self.model_inputs["seq_lens_this_time"], |
| 996 | self.model_inputs, |
| 997 | True, |
| 998 | self.cache_config.block_size, |
| 999 | self.model_inputs["draft_tokens"], |
| 1000 | self.model_inputs["seq_lens_encoder"], |
| 1001 | self.model_inputs["seq_lens_decoder"], |
| 1002 | ) |
| 1003 | self._initialize_forward_meta_xpu() |
| 1004 | # Get sampling metadata |
| 1005 | self.sampling_metadata = SamplingMetadata( |
| 1006 | temperature=self.model_inputs["temperature"], |
| 1007 | top_p=self.model_inputs["top_p"], |
| 1008 | top_k=self.model_inputs["top_k"], |
| 1009 | seed=self.model_inputs["infer_seed"], |
| 1010 | step_idx=self.model_inputs["step_idx"], |
| 1011 | pre_token_ids=self.model_inputs["pre_ids"], |
| 1012 | frequency_penalties=self.model_inputs["frequency_score"], |
| 1013 | presence_penalties=self.model_inputs["presence_score"], |
| 1014 | repetition_penalties=self.model_inputs["penalty_score"], |
| 1015 | min_dec_lens=self.model_inputs["min_dec_len"], |
| 1016 | bad_words_token_ids=self.model_inputs["bad_tokens"], |
| 1017 | eos_token_ids=self.model_inputs["eos_token_id"], |
| 1018 | max_num_logprobs=20 if self.enable_logprob else None, |
| 1019 | temp_scaled_logprobs=self.model_inputs["temp_scaled_logprobs"], |
| 1020 | top_p_normalized_logprobs=self.model_inputs["top_p_normalized_logprobs"], |
| 1021 | share_inputs=self.model_inputs, |
| 1022 | ) |
| 1023 | |
| 1024 | if self.num_model_steps > 1: |
| 1025 | self.model_inputs.last_seq_lens_this_time = paddle.clone(self.model_inputs["seq_lens_this_time"]) |
| 1026 | |
| 1027 | model_output = self.model( |
| 1028 | ids_remove_padding=self.model_inputs["ids_remove_padding"], |
| 1029 | previous_hidden_states=self.model_inputs["target_hidden_states"], |
| 1030 | forward_meta=self.forward_meta, |
| 1031 | ) |
| 1032 | hidden_states = xpu_process_output( |
| 1033 | model_output, self.model_inputs["cum_offsets"], self.forward_meta, self.model_inputs |
| 1034 | ) |
| 1035 | # 4. Compute logits, Sample |
| 1036 | logits = self.model.compute_logits(hidden_states, forward_meta=self.forward_meta) |
| 1037 | sampled_token_ids, sampler_output = self.sampler( |
| 1038 | logits, |
nothing calls this directly
no test coverage detected