MCPcopy Create free account
hub / github.com/cockroachdb/apd / quantize

Method quantize

context.go:1171–1216  ·  view source on GitHub ↗
(d, v *Decimal, exp int32)

Source from the content-addressed store, hash-verified

1169}
1170
1171func (c *Context) quantize(d, v *Decimal, exp int32) Condition {
1172 diff := exp - v.Exponent
1173 d.Set(v)
1174 var res Condition
1175 if diff < 0 {
1176 if diff < MinExponent {
1177 return SystemUnderflow | Underflow
1178 }
1179 var tmpE BigInt
1180 d.Coeff.Mul(&d.Coeff, tableExp10(-int64(diff), &tmpE))
1181 } else if diff > 0 {
1182 p := int32(d.NumDigits()) - diff
1183 if p < 0 {
1184 if !d.IsZero() {
1185 d.Coeff.SetInt64(0)
1186 res = Inexact | Rounded
1187 }
1188 } else {
1189 nc := c.WithPrecision(uint32(p))
1190
1191 // The idea here is that the resulting d.Exponent after rounding will be 0. We
1192 // have a number of, say, 5 digits, but p (our precision) above is set at, say,
1193 // 3. So here d.Exponent is set to `-2`. We have a number like `NNN.xx`, where
1194 // the `.xx` part will be rounded away. However during rounding of 0.9 to 1.0,
1195 // d.Exponent could be set to 1 instead of 0, so we have to reduce it and
1196 // increase the coefficient below.
1197
1198 // Another solution is to set d.Exponent = v.Exponent and adjust it to exp,
1199 // instead of setting d.Exponent = -diff and adjusting it to zero. Although
1200 // this computes the correct result, it fails the Max/MinExponent checks
1201 // during Round and raises underflow flags. Quantize (as per the spec)
1202 // is guaranteed to not raise underflow, and using 0 instead of exp as the
1203 // target eliminates this problem.
1204
1205 d.Exponent = -diff
1206 // Round even if nc.Precision == 0.
1207 res = nc.Rounding.Round(nc, d, d, false /* disableIfPrecisionZero */)
1208 // Adjust for 0.9 -> 1.0 rollover.
1209 if d.Exponent > 0 {
1210 d.Coeff.Mul(&d.Coeff, bigTen)
1211 }
1212 }
1213 }
1214 d.Exponent = exp
1215 return res
1216}
1217
1218func (c *Context) toIntegral(d, x *Decimal) Condition {
1219 res := c.quantize(d, x, 0)

Callers 3

PowMethod · 0.95
QuantizeMethod · 0.95
toIntegralMethod · 0.95

Calls 8

WithPrecisionMethod · 0.95
tableExp10Function · 0.85
NumDigitsMethod · 0.80
IsZeroMethod · 0.80
SetMethod · 0.45
MulMethod · 0.45
SetInt64Method · 0.45
RoundMethod · 0.45

Tested by

no test coverage detected