Function increment_utf8

datafusion/pruning/src/pruning_predicate.rs:1963–2000

Increment a UTF8 string by one, returning `None` if it can't be incremented. This makes it so that the returned string will always compare greater than the input string or any other string with the same prefix. This is necessary since the statistics may have been truncated: if we have a min statistic of "fo" that may have originally been "foz" or anything else with the prefix "fo". E.g. `increment

(data: &str)

/// E.g. `increment_utf8("foo") >= "foo"` and `increment_utf8("foo") >= "fooz"`
/// In this example `increment_utf8("foo") == "fop"
fn increment_utf8(data: &str) -> Option<String> {
    // Helper function to check if a character is valid to use
    fn is_valid_unicode(c: char) -> bool {
        let cp = c as u32;

        // Filter out non-characters (https://www.unicode.org/versions/corrigendum9.html)
        if [0xFFFE, 0xFFFF].contains(&cp) || (0xFDD0..=0xFDEF).contains(&cp) {
            return false;
        }

        // Filter out private use area
        if cp >= 0x110000 {
            return false;
        }

        true
    }

    // Convert string to vector of code points
    let mut code_points: Vec<char> = data.chars().collect();

    // Work backwards through code points
    for idx in (0..code_points.len()).rev() {
        let original = code_points[idx] as u32;

        // Try incrementing the code point
        if let Some(next_char) = char::from_u32(original + 1)
            && is_valid_unicode(next_char)
        {
            code_points[idx] = next_char;
            // truncate the string to the current index
            code_points.truncate(idx + 1);
            return Some(code_points.into_iter().collect());
        }
    }

    None
}

/// Wrap the statistics expression in a check that skips the expression if the column is all nulls.
///