calculate some parameters
(self)
| 1882 | ) |
| 1883 | |
| 1884 | def postprocess(self): |
| 1885 | """ |
| 1886 | calculate some parameters |
| 1887 | """ |
| 1888 | # Unified field model config |
| 1889 | if self.model_config.architectures[0] == "Glm4MoeForCausalLM": |
| 1890 | # The first moe layer id of GLM4.5 model |
| 1891 | self.model_config.moe_layer_start_index = self.model_config.first_k_dense_replace |
| 1892 | |
| 1893 | if self.parallel_config.tensor_parallel_size <= self.worker_num_per_node or self.node_rank == 0: |
| 1894 | self.is_master = True |
| 1895 | self.master_ip = "0.0.0.0" |
| 1896 | else: |
| 1897 | self.is_master = False |
| 1898 | self.master_ip = self.ips[0] |
| 1899 | |
| 1900 | self.paddle_commit_id = paddle.version.commit |
| 1901 | |
| 1902 | if self.scheduler_config.max_num_batched_tokens is None: |
| 1903 | if int(envs.ENABLE_V1_KVCACHE_SCHEDULER): |
| 1904 | if paddle.is_compiled_with_xpu(): |
| 1905 | self.scheduler_config.max_num_batched_tokens = self.model_config.max_model_len |
| 1906 | else: |
| 1907 | self.scheduler_config.max_num_batched_tokens = 8192 # if set to max_model_len, it's easy to be OOM |
| 1908 | else: |
| 1909 | if self.cache_config.enable_chunked_prefill: |
| 1910 | self.scheduler_config.max_num_batched_tokens = 2048 |
| 1911 | else: |
| 1912 | self.scheduler_config.max_num_batched_tokens = self.model_config.max_model_len |
| 1913 | |
| 1914 | if self.long_prefill_token_threshold == 0: |
| 1915 | self.long_prefill_token_threshold = int(self.model_config.max_model_len * 0.04) |
| 1916 | |
| 1917 | self.cache_config.max_block_num_per_seq = int(self.model_config.max_model_len // self.cache_config.block_size) |
| 1918 | self.cache_config.postprocess(self.get_max_chunk_tokens(), self.scheduler_config.max_num_seqs) |
| 1919 | if self.model_config is not None and self.model_config.enable_mm and not envs.ENABLE_V1_KVCACHE_SCHEDULER: |
| 1920 | self.cache_config.enable_prefix_caching = False |
| 1921 | if ( |
| 1922 | self.structured_outputs_config is not None |
| 1923 | and self.structured_outputs_config.guided_decoding_backend != "off" |
| 1924 | ): |
| 1925 | if current_platform.is_xpu() or self.speculative_config.method is not None: |
| 1926 | logger.warning("Speculative Decoding and XPU currently do not support Guided decoding, set off.") |
| 1927 | self.structured_outputs_config.guided_decoding_backend = "off" |
| 1928 | elif self.structured_outputs_config.guided_decoding_backend in ["auto", "xgrammar"]: |
| 1929 | self.structured_outputs_config.guided_decoding_backend = "xgrammar" |
| 1930 | elif self.structured_outputs_config.guided_decoding_backend == "guidance": |
| 1931 | try: |
| 1932 | import llguidance.torch |
| 1933 | |
| 1934 | llguidance.torch |
| 1935 | except ImportError: |
| 1936 | raise ImportError( |
| 1937 | "The 'llguidance' package is required for using guidance as the guided decoding backend. " |
| 1938 | "Please install it via the appropriate method." |
| 1939 | ) |
| 1940 | else: |
| 1941 | raise NotImplementedError( |