MCPcopy Create free account
hub / github.com/DeepRec-AI/DeepRec / _compute_quantization_error

Function _compute_quantization_error

tools/low_precision_optimize/calibrate.py:31–71  ·  view source on GitHub ↗

Compute the quantization error if we use start_bin to end_bin as the min and max to do the quantization.

(next_start_bin, next_end_bin, norm_type)

Source from the content-addressed store, hash-verified

29 return density * norm
30
31 def _compute_quantization_error(next_start_bin, next_end_bin, norm_type):
32 """
33 Compute the quantization error if we use start_bin to end_bin as the
34 min and max to do the quantization.
35 """
36 dst_bin_width = bin_width * (next_end_bin - next_start_bin + 1) / dst_nbins
37 if dst_bin_width == 0.0:
38 return 0.0
39
40 src_bin = np.arange(bins)
41 # distances from the beginning of first dst_bin to the beginning and
42 # end of src_bin
43 src_bin_begin = (src_bin - next_start_bin) * bin_width
44 src_bin_end = src_bin_begin + bin_width
45
46 # which dst_bins the beginning and end of src_bin belong to?
47 dst_bin_of_begin = np.clip(src_bin_begin // dst_bin_width, 0, dst_nbins - 1)
48 dst_bin_of_begin_center = (dst_bin_of_begin + 0.5) * dst_bin_width
49
50 dst_bin_of_end = np.clip(src_bin_end // dst_bin_width, 0, dst_nbins - 1)
51 dst_bin_of_end_center = (dst_bin_of_end + 0.5) * dst_bin_width
52
53 density = histogram / bin_width
54
55 norm = np.zeros(bins)
56
57 delta_begin = src_bin_begin - dst_bin_of_begin_center
58 delta_end = dst_bin_width / 2
59 norm += _get_norm(delta_begin, delta_end, density, norm_type)
60
61 norm += (dst_bin_of_end - dst_bin_of_begin - 1) * _get_norm(
62 -dst_bin_width / 2, dst_bin_width / 2, density, norm_type
63 )
64
65 dst_bin_of_end_center = dst_bin_of_end * dst_bin_width + dst_bin_width / 2
66
67 delta_begin = -dst_bin_width / 2
68 delta_end = src_bin_end - dst_bin_of_end_center
69 norm += _get_norm(delta_begin, delta_end, density, norm_type)
70
71 return np.sum(norm)
72
73 # cumulative sum
74 total = sum(histogram)

Callers 1

Calls 2

_get_normFunction · 0.85
sumMethod · 0.45

Tested by

no test coverage detected