Function general_except

datafusion/functions-nested/src/except.rs:165–239

(
    l: &GenericListArray<OffsetSize>,
    r: &GenericListArray<OffsetSize>,
    field: &FieldRef,
)

}

fn general_except<OffsetSize: OffsetSizeTrait>(
    l: &GenericListArray<OffsetSize>,
    r: &GenericListArray<OffsetSize>,
    field: &FieldRef,
) -> Result<GenericListArray<OffsetSize>> {
    let converter = RowConverter::new(vec![SortField::new(l.value_type())])?;

    // Only convert the visible portion of the values array. For sliced
    // ListArrays, values() returns the full underlying array but only
    // elements between the first and last offset are referenced.
    let l_first = l.offsets()[0].as_usize();
    let l_len = l.offsets()[l.len()].as_usize() - l_first;
    let l_values = converter.convert_columns(&[l.values().slice(l_first, l_len)])?;

    let r_first = r.offsets()[0].as_usize();
    let r_len = r.offsets()[r.len()].as_usize() - r_first;
    let r_values = converter.convert_columns(&[r.values().slice(r_first, r_len)])?;

    let mut offsets = Vec::<OffsetSize>::with_capacity(l.len() + 1);
    offsets.push(OffsetSize::usize_as(0));

    let mut indices: Vec<usize> = Vec::with_capacity(l_values.num_rows());
    let mut dedup = HashSet::new();

    let nulls = NullBuffer::union(l.nulls(), r.nulls());

    let l_offsets_iter = l.offsets().iter().tuple_windows();
    let r_offsets_iter = r.offsets().iter().tuple_windows();
    for (list_index, ((l_start, l_end), (r_start, r_end))) in
        l_offsets_iter.zip(r_offsets_iter).enumerate()
    {
        if nulls
            .as_ref()
            .is_some_and(|nulls| nulls.is_null(list_index))
        {
            offsets.push(OffsetSize::usize_as(indices.len()));
            continue;
        }

        for element_index in r_start.as_usize() - r_first..r_end.as_usize() - r_first {
            let right_row = r_values.row(element_index);
            dedup.insert(right_row);
        }
        for element_index in l_start.as_usize() - l_first..l_end.as_usize() - l_first {
            let left_row = l_values.row(element_index);
            if dedup.insert(left_row) {
                indices.push(element_index + l_first);
            }
        }

        offsets.push(OffsetSize::usize_as(indices.len()));
        dedup.clear();
    }

    // Gather distinct left-side values by index.
    // Use UInt64Array for LargeList to support values arrays exceeding u32::MAX.
    let values = if indices.is_empty() {
        arrow::array::new_empty_array(&l.value_type())