Function quantize_8bits

src/quantization.cpp:88–174

}

static void quantize_8bits(program& prog,
                           const target& t,
                           shape::type_t precision,
                           const std::vector<parameter_map>& calibration,
                           const std::unordered_set<std::string>& ins_names)
{
    // Run optimize_module() before converting to int8/fp8 to const eval and fold in FP32 to
    // avoid loss of precision.
    run_passes(prog, {rewrite_rnn{}, normalize_ops{}, optimize_module{}}, quant_tracer());

    std::shared_ptr<std::vector<std::pair<float, float>>> quant_8bit_params =
        std::make_shared<std::vector<std::pair<float, float>>>();
    std::shared_ptr<std::vector<float>> max_abs_vals = std::make_shared<std::vector<float>>();
    std::map<shape::type_t, float> type_ranges       = {{shape::type_t::int8_type, 127.0},
                                                        {shape::type_t::fp8e4m3fnuz_type, 240.0},
                                                        {shape::type_t::fp8e4m3fn_type, 448.0}};
    float quantized_range                            = type_ranges.at(precision);
    auto calc_quant_params = [&](std::size_t ins_index, std::vector<argument> args) {
        std::pair<float, float> param_pair{64.0f, 0.0f};
        // scale and shift is need for only int8 type, and we do not
        // consider shift, so set shift to 0
        std::vector<float> vec_val;
        argument arg = t.copy_from(args.front());
        arg.visit([&](auto output) { vec_val.assign(output.begin(), output.end()); });
        auto max_val                = *std::max_element(vec_val.begin(), vec_val.end());
        auto min_val                = *std::min_element(vec_val.begin(), vec_val.end());
        auto max_abs                = std::max(std::fabs(max_val), std::fabs(min_val));
        max_abs_vals->at(ins_index) = std::max(max_abs_vals->at(ins_index), max_abs);
        // if all values are 0, no need to do scaling
        if(float_equal(max_abs_vals->at(ins_index), 0.0f))
        {
            param_pair.first = 1.0f;
        }
        else
        {
            param_pair.first = quantized_range / max_abs_vals->at(ins_index);
        }
        quant_8bit_params->at(ins_index) = param_pair;
    };

    // pass to add capture argument op
    std::size_t param_num = 0;
    run_passes(
        prog, {capture_arguments_pass{ins_names, calc_quant_params, &param_num}}, quant_tracer());
    quant_8bit_params->resize(param_num, std::pair<float, float>(64.0f, 0.0f));
    max_abs_vals->resize(param_num, 0.0f);

    // use the calibration data to compute the quantization scale
    auto capture_prog = prog;
    capture_prog.compile(t);

    // use all calibration data to run the program to calculate the
    // quantization scale and shift
    for(auto&& arg : calibration)
    {
        parameter_map m;
        for(auto&& x : capture_prog.get_parameter_shapes())
        {