MCPcopy Create free account
hub / github.com/cockroachdb/apd / Quantize

Method Quantize

context.go:1149–1169  ·  view source on GitHub ↗

Quantize adjusts and rounds x as necessary so it is represented with exponent exp and stores the result in d.

(d, x *Decimal, exp int32)

Source from the content-addressed store, hash-verified

1147// Quantize adjusts and rounds x as necessary so it is represented with
1148// exponent exp and stores the result in d.
1149func (c *Context) Quantize(d, x *Decimal, exp int32) (Condition, error) {
1150 if c.shouldSetAsNaN(x, nil) {
1151 return c.setAsNaN(d, x, nil)
1152 }
1153 if x.Form == Infinite || exp < c.etiny() {
1154 d.Set(decimalNaN)
1155 return c.goError(InvalidOperation)
1156 }
1157 res := c.quantize(d, x, exp)
1158 if nd := d.NumDigits(); nd > int64(c.Precision) || exp > c.MaxExponent {
1159 res = InvalidOperation
1160 d.Set(decimalNaN)
1161 } else {
1162 res |= c.round(d, d)
1163 if res.Overflow() || res.Underflow() {
1164 res = InvalidOperation
1165 d.Set(decimalNaN)
1166 }
1167 }
1168 return c.goError(res)
1169}
1170
1171func (c *Context) quantize(d, v *Decimal, exp int32) Condition {
1172 diff := exp - v.Exponent

Callers 3

RunMethod · 0.45
ExampleContext_QuantizeFunction · 0.45
TestQuantizeFunction · 0.45

Calls 10

shouldSetAsNaNMethod · 0.95
setAsNaNMethod · 0.95
etinyMethod · 0.95
goErrorMethod · 0.95
quantizeMethod · 0.95
roundMethod · 0.95
NumDigitsMethod · 0.80
OverflowMethod · 0.80
UnderflowMethod · 0.80
SetMethod · 0.45

Tested by 3

RunMethod · 0.36
ExampleContext_QuantizeFunction · 0.36
TestQuantizeFunction · 0.36