Function string_to_array_scalar_args

datafusion/functions-nested/src/string.rs:335–415

Optimized `string_to_array` implementation for the common case where delimiter and null_value are scalar values.

(
    string_array: &StringArrType,
    delimiter: Option<&str>,
    null_value: Option<&str>,
    string_builder: StringBuilderType,
)

/// Optimized `string_to_array` implementation for the common case where
/// delimiter and null_value are scalar values.
fn string_to_array_scalar_args<'a, StringArrType, StringBuilderType>(
    string_array: &StringArrType,
    delimiter: Option<&str>,
    null_value: Option<&str>,
    string_builder: StringBuilderType,
) -> Result<ArrayRef>
where
    StringArrType: StringArrayType<'a>,
    StringBuilderType: StringArrayBuilderType,
{
    let mut list_builder = ListBuilder::new(string_builder);

    match delimiter {
        Some("") => {
            // Empty delimiter: each non-empty string becomes a single-element list.
            // Empty strings produce an empty array (PostgreSQL compat).
            for i in 0..string_array.len() {
                if string_array.is_null(i) {
                    list_builder.append(false);
                    continue;
                }
                let string = string_array.value(i);
                if !string.is_empty() {
                    append_part(list_builder.values(), string, null_value);
                }
                list_builder.append(true);
            }
        }
        Some(delimiter) => {
            // Rather than using `str::split`, do the split ourselves using
            // `memmem::Finder`. This allows pre-compiling the delimiter search
            // pattern once and reusing it for all rows.
            let finder = memchr::memmem::Finder::new(delimiter.as_bytes());
            let delim_len = delimiter.len();

            for i in 0..string_array.len() {
                if string_array.is_null(i) {
                    list_builder.append(false);
                    continue;
                }
                let string = string_array.value(i);
                if !string.is_empty() {
                    let bytes = string.as_bytes();
                    let mut start = 0;
                    for pos in finder.find_iter(bytes) {
                        append_part(
                            list_builder.values(),
                            &string[start..pos],
                            null_value,
                        );
                        start = pos + delim_len;
                    }
                    // Trailing part after last delimiter (or entire string if no
                    // delimiter was found).
                    append_part(list_builder.values(), &string[start..], null_value);
                }
                list_builder.append(true);
            }