Initialize forward meta and attention meta data
(self, step_use_cudagraph: bool = False, is_dummy_run: bool = False, substep: int = 0)
| 621 | self.model_inputs.seq_lens_this_time = self.model_inputs["seq_lens_this_time_buffer"] |
| 622 | |
| 623 | def _initialize_forward_meta(self, step_use_cudagraph: bool = False, is_dummy_run: bool = False, substep: int = 0): |
| 624 | """ |
| 625 | Initialize forward meta and attention meta data |
| 626 | """ |
| 627 | # Initialize forward meta |
| 628 | self.forward_meta = ForwardMeta( |
| 629 | ids_remove_padding=self.model_inputs["ids_remove_padding"], |
| 630 | rotary_embs=self.model_inputs["rope_emb"], |
| 631 | attn_backend=self.attn_backends[0], |
| 632 | decoder_batch_ids=self.model_inputs["decoder_batch_ids"], |
| 633 | decoder_tile_ids_per_batch=self.model_inputs["decoder_tile_ids_per_batch"], |
| 634 | decoder_num_blocks_cpu=self.model_inputs["decoder_num_blocks_cpu"], |
| 635 | decoder_num_blocks_device=self.model_inputs["decoder_num_blocks_device"], |
| 636 | decoder_chunk_size_device=self.model_inputs["decoder_chunk_size_device"], |
| 637 | max_len_tensor_cpu=self.model_inputs["max_len_tensor_cpu"], |
| 638 | seq_lens_encoder=self.model_inputs["seq_lens_encoder"], |
| 639 | seq_lens_decoder=self.model_inputs["seq_lens_decoder"], |
| 640 | seq_lens_this_time=self.model_inputs["seq_lens_this_time"], |
| 641 | batch_id_per_token=self.model_inputs["batch_id_per_token"], |
| 642 | cu_seqlens_q=self.model_inputs["cu_seqlens_q"], |
| 643 | cu_seqlens_k=self.model_inputs["cu_seqlens_k"], |
| 644 | block_tables=self.model_inputs["block_tables"], |
| 645 | caches=self.model_inputs["caches"], |
| 646 | encoder_batch_ids=self.model_inputs["encoder_batch_ids"], |
| 647 | encoder_tile_ids_per_batch=self.model_inputs["encoder_tile_ids_per_batch"], |
| 648 | encoder_num_blocks_x_cpu=self.model_inputs["encoder_num_blocks_x_cpu"], |
| 649 | kv_batch_ids=self.model_inputs["kv_batch_ids"], |
| 650 | kv_tile_ids_per_batch=self.model_inputs["kv_tile_ids_per_batch"], |
| 651 | kv_num_blocks_x_cpu=self.model_inputs["kv_num_blocks_x_cpu"], |
| 652 | attn_mask_offsets=self.model_inputs["attn_mask_offsets"] if self.enable_mm else None, |
| 653 | ) |
| 654 | |
| 655 | # Initialzie attention meta data |
| 656 | for attn_backend in self.attn_backends: |
| 657 | attn_backend.init_attention_metadata(self.forward_meta) |
| 658 | |
| 659 | # Notes(liuzichang): |
| 660 | # 1. CUDA Graph capture sizes must be recorded in descending order (large → small). |
| 661 | # 2. In multi-step execution, only the first step should be captured. |
| 662 | self.forward_meta.step_use_cudagraph = ( |
| 663 | step_use_cudagraph and self.draft_model_use_cudagraph and not (substep > 0 and is_dummy_run) |
| 664 | ) |
| 665 | |
| 666 | def _initialize_forward_meta_xpu(self): |
| 667 |