(model, fd_config: FDConfig)
| 243 | |
| 244 | |
| 245 | def process_final_after_loading(model, fd_config: FDConfig): |
| 246 | # process_final_after_loading handles the post-loading process for cases other than dynamic quantization. |
| 247 | from fastdeploy.model_executor.layers.linear import ( |
| 248 | KVBatchLinear, |
| 249 | UnquantizedLinearMethod, |
| 250 | ) |
| 251 | from fastdeploy.model_executor.layers.moe.moe import get_moe_method |
| 252 | |
| 253 | for name, sublayer in model.named_sublayers(): |
| 254 | if isinstance(sublayer, KVBatchLinear): |
| 255 | continue |
| 256 | quant_method = getattr(sublayer, "quant_method", None) |
| 257 | if quant_method is not None: |
| 258 | unquant_moe_layer = get_moe_method() |
| 259 | if unquant_moe_layer is None: |
| 260 | unquant_moe_cls = object |
| 261 | else: |
| 262 | unquant_moe_cls = type(unquant_moe_layer) |
| 263 | is_unquant_cls = type(quant_method) is UnquantizedLinearMethod or type(quant_method) is unquant_moe_cls |
| 264 | is_offline_quantized_ckpt = not (fd_config.quant_config and fd_config.quant_config.is_checkpoint_bf16) |
| 265 | if is_unquant_cls or is_offline_quantized_ckpt: |
| 266 | if hasattr(quant_method, "process_weights_after_loading"): |
| 267 | quant_method.process_weights_after_loading(sublayer) |
| 268 | continue |
| 269 | if not hasattr(sublayer, "process_weights_after_loading"): |
| 270 | continue |
| 271 | sublayer.process_weights_after_loading() |
| 272 | |
| 273 | |
| 274 | def free_tensor(tensor): |
no test coverage detected