Function convert

neural_compressor/torch/quantization/quantize.py:253–325

Convert the prepared model to a quantized model. Args: model (torch.nn.Module): torch model quant_config (BaseConfig, optional): path to quantization config, only required when model is not prepared. inplace (bool, optional): It will change the given model in-place if Tr

(
    model: torch.nn.Module,
    quant_config: BaseConfig = None,
    inplace: bool = True,
    **kwargs,
)

@log_process(mode=Mode.CONVERT)
def convert(
    model: torch.nn.Module,
    quant_config: BaseConfig = None,
    inplace: bool = True,
    **kwargs,
):
    """Convert the prepared model to a quantized model.

    Args:
        model (torch.nn.Module): torch model
        quant_config (BaseConfig, optional): path to quantization config, only required when model is not prepared.
        inplace (bool, optional): It will change the given model in-place if True.

    Returns:
        The quantized model.
    """
    if isinstance(model, _AutoRoundModelReference):
        if quant_config is None:
            quant_config = model.quant_config
        else:
            logger.warning("quant_config will be ignored since the model has been prepared.")
        model = model.model_reference

    q_model = model if inplace else copy.deepcopy(model)

    assert (
        getattr(model, "is_prepared", False) or quant_config is not None
    ), "Please pass quant_config to convert function."

    if getattr(model, "is_prepared", False):
        if quant_config is None:
            quant_config = model.quant_config
        else:
            logger.warning("quant_config will be ignored since the model has been prepared.")
    example_inputs = model.example_inputs if getattr(model, "is_prepared", False) else None

    registered_configs = config_registry.get_cls_configs()
    if isinstance(quant_config, dict):
        quant_config = ComposableConfig.from_dict(quant_config, config_registry=registered_configs[FRAMEWORK_NAME])
        logger.info(f"Parsed a config dict to construct the quantization config: {quant_config}.")
    else:
        assert isinstance(
            quant_config, BaseConfig
        ), f"Please pass a dict or config instance as the quantization configuration, but got {type(quant_config)}."
    if hasattr(model, "dpq_quantized") and model.dpq_quantized and type(quant_config) == FP8Config:
        quant_config = HybridGPTQConfig.convert_from_fp8(quant_config)
    logger.debug("Convert model with config:")
    logger.debug(quant_config.to_dict())

    # select quantization algo according to config
    if is_ipex_available and (
        isinstance(quant_config, INT8StaticQuantConfig) or isinstance(quant_config, SmoothQuantConfig)
    ):
        model_info = quant_config.get_model_info(q_model, example_inputs)
    else:
        model_info = quant_config.get_model_info(model=q_model)
    configs_mapping = quant_config.to_config_mapping(model_info=model_info)
    logger.debug(configs_mapping)