Configuration for compute graph level optimization.
| 889 | |
| 890 | |
| 891 | class GraphOptimizationConfig: |
| 892 | """ |
| 893 | Configuration for compute graph level optimization. |
| 894 | """ |
| 895 | |
| 896 | def __init__( |
| 897 | self, |
| 898 | args, |
| 899 | ): |
| 900 | """The Top-level graph optimization contral corresponds to different backends. |
| 901 | - 0: dyncmic graph |
| 902 | - 1: static graph |
| 903 | - 2: static graph + cinn compilation backend |
| 904 | """ |
| 905 | self.graph_opt_level: int = 0 |
| 906 | |
| 907 | # CUDA Graph Config |
| 908 | """ Whether to use cudagraph. |
| 909 | - False: cudagraph is not used. |
| 910 | - True: cudagraph is used. |
| 911 | It requires that all input buffers have fixed addresses, and all |
| 912 | splitting ops write their outputs to input buffers. |
| 913 | - With dyncmic graph backend: ... |
| 914 | - With static graph backend: WIP |
| 915 | """ |
| 916 | self.sot_warmup_sizes: list[int] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 32, 64, 128] |
| 917 | """ Number of warmup runs for SOT warmup. """ |
| 918 | self.use_cudagraph: bool = False if paddle.is_compiled_with_xpu() else True |
| 919 | """Sizes to capture cudagraph. |
| 920 | - None (default): capture sizes are inferred from llm config. |
| 921 | - list[int]: capture sizes are specified as given.""" |
| 922 | self.cudagraph_capture_sizes: Optional[list[int]] = None |
| 923 | self.cudagraph_capture_sizes_prefill: list[int] = [1, 2, 4, 8] |
| 924 | """ Number of warmup runs for cudagraph. """ |
| 925 | self.cudagraph_num_of_warmups: int = 2 |
| 926 | """Whether to copy input tensors for cudagraph. |
| 927 | If the caller can guarantee that the same input buffers |
| 928 | are always used, it can set this to False. Otherwise, it should |
| 929 | set this to True.""" |
| 930 | self.cudagraph_copy_inputs: bool = False |
| 931 | """ In static graph, this is an operation list that does not need to be captured by the CUDA graph. |
| 932 | CudaGraphBackend will split these operations from the static graph. |
| 933 | Example usage: |
| 934 | cudagraph_splitting_ops = ["paddle.unified_attention"] |
| 935 | |
| 936 | Note: If want to use subgraph capture functionality in a dynamic graph, |
| 937 | can manually split the model into multiple layers and apply the @support_graph_optimization decorator |
| 938 | only to the layer where CUDA graph functionality is required. |
| 939 | """ |
| 940 | self.cudagraph_splitting_ops: list[str] = [] |
| 941 | """ Whether to use a full cuda graph for the entire forward pass rather than |
| 942 | splitting certain operations such as attention into subgraphs. |
| 943 | Thus this flag cannot be used together with splitting_ops.""" |
| 944 | self.cudagraph_only_prefill: bool = False |
| 945 | """When cudagraph_only_prefill is False, only capture decode-only. |
| 946 | When cudagraph_only_prefill is True, only capture prefill-only. |
| 947 | Now don't support capture both decode-only and prefill-only""" |
| 948 | self.full_cuda_graph: bool = True |
no outgoing calls