Function compact_list

datafusion/functions-nested/src/array_compact.rs:128–191

Remove null elements from each row of a list array.

(
    list_array: &GenericListArray<O>,
    field: &Arc<arrow::datatypes::Field>,
)

/// Remove null elements from each row of a list array.
fn compact_list<O: OffsetSizeTrait>(
    list_array: &GenericListArray<O>,
    field: &Arc<arrow::datatypes::Field>,
) -> Result<ArrayRef> {
    let values = list_array.values();

    // Fast path: no nulls in values, return input unchanged
    if values.null_count() == 0 {
        return Ok(Arc::new(list_array.clone()));
    }

    let original_data = values.to_data();
    let capacity = original_data.len() - values.null_count();
    let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
    offsets.push(O::zero());
    let mut mutable = MutableArrayData::with_capacities(
        vec![&original_data],
        false,
        Capacities::Array(capacity),
    );

    for row_index in 0..list_array.len() {
        if list_array.nulls().is_some_and(|n| n.is_null(row_index)) {
            offsets.push(offsets[row_index]);
            continue;
        }

        let start = list_array.offsets()[row_index].as_usize();
        let end = list_array.offsets()[row_index + 1].as_usize();
        let mut copied = 0usize;

        // Batch consecutive non-null elements into single extend() calls
        // to reduce per-element overhead. For [1, 2, NULL, 3, 4] this
        // produces 2 extend calls (0..2, 3..5) instead of 4 individual ones.
        let mut batch_start: Option<usize> = None;
        for i in start..end {
            if values.is_null(i) {
                // Null breaks the current batch — flush it
                if let Some(bs) = batch_start {
                    mutable.extend(0, bs, i);
                    copied += i - bs;
                    batch_start = None;
                }
            } else if batch_start.is_none() {
                batch_start = Some(i);
            }
        }
        // Flush any remaining batch after the loop
        if let Some(bs) = batch_start {
            mutable.extend(0, bs, end);
            copied += end - bs;
        }

        offsets.push(offsets[row_index] + O::usize_as(copied));
    }

    let new_values = make_array(mutable.freeze());
    Ok(Arc::new(GenericListArray::<O>::try_new(