Configuration for speculative decoding.
| 716 | |
| 717 | |
| 718 | class SpeculativeConfig: |
| 719 | """ |
| 720 | Configuration for speculative decoding. |
| 721 | """ |
| 722 | |
| 723 | def __init__( |
| 724 | self, |
| 725 | args, |
| 726 | ): |
| 727 | self.method_list = ["ngram_match", "mtp", "suffix"] |
| 728 | self.mtp_strategy_list = ["default", "with_ngram"] |
| 729 | |
| 730 | # speculative method, choose in [None, "ngram_match", "mtp", "hybrid_mtp_ngram"] |
| 731 | self.method: Optional[str] = None |
| 732 | # mtp strategy in mtp-method |
| 733 | self.mtp_strategy = "default" |
| 734 | # the max length of speculative tokens |
| 735 | self.num_speculative_tokens: int = 1 |
| 736 | # the model runner step of draft model/mtp... |
| 737 | self.num_model_steps: int = 1 |
| 738 | # the max length of candidate tokens for speculative method |
| 739 | self.max_candidate_len: int = 5 |
| 740 | # the max length of verify window for speculative method |
| 741 | self.verify_window: int = 2 |
| 742 | # ngram match |
| 743 | self.max_ngram_size: int = 5 |
| 744 | self.min_ngram_size: int = 2 |
| 745 | # Suffix Decoding |
| 746 | # The maximum length of token sequences cached in suffix trees. |
| 747 | self.suffix_decoding_max_tree_depth: int = 64 |
| 748 | # The limits of requests that can be stored in the cache. |
| 749 | self.suffix_decoding_max_cached_requests: int = -1 |
| 750 | # The factor of matched length, calculated as num_draft_tokens = suffix_max_spec_factor * matched_length |
| 751 | self.suffix_decoding_max_spec_factor: float = 1.0 |
| 752 | # The probability threshold for speculated tokens. |
| 753 | self.suffix_decoding_min_token_prob: float = 0.1 |
| 754 | # model for mtp/eagle/draft_model |
| 755 | self.model: Optional[str] = None |
| 756 | # quantization of model |
| 757 | self.quantization: Optional[Dict[str, Any]] = None |
| 758 | # allocate more blocks to prevent mtp from finishing the block earlier than the main model |
| 759 | # Fixed now |
| 760 | self.num_gpu_block_expand_ratio: Optional[float] = 1 |
| 761 | # To distinguish the main model and draft model(mtp/eagle/draftmodel) |
| 762 | # ["main", "mtp"] |
| 763 | self.model_type: Optional[str] = "main" |
| 764 | # TODO(liuzichang): To reduce memory usage, MTP shares the main model's lm_head and embedding layers. |
| 765 | # A trick method is currently used to enable this sharing. |
| 766 | # This will be replaced with a more standardized solution in the future. |
| 767 | self.sharing_model = None |
| 768 | # During benchmarking, we need to enforce that the number of accepted tokens is 1. |
| 769 | # This means no tokens from MTP are accepted. |
| 770 | # This ensures that the specified simulation acceptance rate is not affected. |
| 771 | self.benchmark_mode: bool = False |
| 772 | # Enable token constraint enforcement in generation phase |
| 773 | # When enabled, enforces specific tokens after the reasoning phase boundary pattern |
| 774 | self.enf_gen_phase_tag: bool = False |
| 775 |
no outgoing calls