Function getAcceptableTickPrecision

src/util/number.ts:339–369

(
    dataExtent: number[],
    // Typically, `Math.abs(pixelExtent[1] - pixelExtent[0])`.
    pxSpan: number,
    // By default, `1`.
    pxDiffAcceptable: number | NullUndefined
    // Return a precision >= 0
    // This precision can be used in method `round`.
    // Return `NaN` for edge case or illegal inputs. Callers need to handle that.
)

 *  not be covered until necessary.
 */
export function getAcceptableTickPrecision(
    dataExtent: number[],
    // Typically, `Math.abs(pixelExtent[1] - pixelExtent[0])`.
    pxSpan: number,
    // By default, `1`.
    pxDiffAcceptable: number | NullUndefined
    // Return a precision >= 0
    // This precision can be used in method `round`.
    // Return `NaN` for edge case or illegal inputs. Callers need to handle that.
): number {
    const dataSpan = mathAbs(dataExtent[1] - dataExtent[0]);
    if (!isFinite(dataSpan) || dataSpan === 0) {
        return NaN;
    }
    // Formula for choosing an acceptable precision:
    //  Let `pxDiff = abs(dataSpan - round(dataSpan, precision))`.
    //  We require `pxDiff <= dataSpan * pxDiffAcceptable / pxSpan`.
    //  Consider the nature of "round", the max `pxDiff` is: `pow(10, -precision) / 2`,
    //  Hence: `pow(10, -precision) / 2 <= dataSpan * pxDiffAcceptable / pxSpan`
    //  Hence: `precision >= -log10(2 * dataSpan * pxDiffAcceptable / pxSpan)`
    const dataExp2 = mathLog(2 * mathAbs(pxDiffAcceptable || 1) * mathAbs(dataSpan)) / mathLN10;
    const pxExp = mathLog(mathAbs(pxSpan)) / mathLN10;
    // PENDING: Rounding error generally does not matter; do not fix it before `Math.ceil`
    // until bad case occur.
    let precision = mathMax(0, mathCeil(-dataExp2 + pxExp));
    if (!isFinite(precision)) {
        // If dataSpan is near `0`, the result should not be too big or even `Infinity`.
        precision = NaN;
    }
    return precision;
}

/**
 * Get a data of given precision, assuring the sum of percentages