Function adjustLastDigit

fflib/v1/extfloat.go:645–668

adjustLastDigit modifies d = x-currentDiff*ε, to get closest to d = x-targetDiff*ε, without becoming smaller than x-maxDiff*ε. It assumes that a decimal digit is worth ulpDecimal*ε, and that all data is known with a error estimate of ulpBinary*ε.

(d *decimalSlice, currentDiff, targetDiff, maxDiff, ulpDecimal, ulpBinary uint64)

// It assumes that a decimal digit is worth ulpDecimal*ε, and that
// all data is known with a error estimate of ulpBinary*ε.
func adjustLastDigit(d *decimalSlice, currentDiff, targetDiff, maxDiff, ulpDecimal, ulpBinary uint64) bool {
	if ulpDecimal < 2*ulpBinary {
		// Approximation is too wide.
		return false
	}
	for currentDiff+ulpDecimal/2+ulpBinary < targetDiff {
		d.d[d.nd-1]--
		currentDiff += ulpDecimal
	}
	if currentDiff+ulpDecimal <= targetDiff+ulpDecimal/2+ulpBinary {
		// we have two choices, and don't know what to do.
		return false
	}
	if currentDiff < ulpBinary || currentDiff > maxDiff-ulpBinary {
		// we went too far
		return false
	}
	if d.nd == 1 && d.d[0] == '0' {
		// the number has actually reached zero.
		d.nd = 0
		d.dp = 0
	}
	return true
}