Function general_array_distinct

datafusion/functions-nested/src/set_ops.rs:547–618

(
    array: &GenericListArray<OffsetSize>,
    field: &FieldRef,
)

}

fn general_array_distinct<OffsetSize: OffsetSizeTrait>(
    array: &GenericListArray<OffsetSize>,
    field: &FieldRef,
) -> Result<ArrayRef> {
    if array.is_empty() {
        return Ok(Arc::new(array.clone()) as ArrayRef);
    }
    let value_offsets = array.value_offsets();
    let dt = array.value_type();
    let mut offsets = Vec::with_capacity(array.len() + 1);
    offsets.push(OffsetSize::usize_as(0));

    let converter = RowConverter::new(vec![SortField::new(dt.clone())])?;

    // Only convert the visible portion of the values array. For sliced
    // ListArrays, values() returns the full underlying array but only
    // elements between the first and last offset are referenced.
    let first_offset = value_offsets[0].as_usize();
    let visible_len = value_offsets[array.len()].as_usize() - first_offset;
    let rows =
        converter.convert_columns(&[array.values().slice(first_offset, visible_len)])?;

    let mut indices: Vec<usize> = Vec::with_capacity(rows.num_rows());
    let mut seen = HashSet::new();
    for i in 0..array.len() {
        let last_offset = *offsets.last().unwrap();

        // Null list entries produce no output; just carry forward the offset.
        if array.is_null(i) {
            offsets.push(last_offset);
            continue;
        }

        let start = value_offsets[i].as_usize() - first_offset;
        let end = value_offsets[i + 1].as_usize() - first_offset;
        seen.clear();
        seen.reserve(end - start);

        // Walk the sub-array and keep only the first occurrence of each value.
        for idx in start..end {
            let row = rows.row(idx);
            if seen.insert(row) {
                indices.push(idx + first_offset);
            }
        }
        offsets.push(last_offset + OffsetSize::usize_as(seen.len()));
    }

    // Gather distinct values in a single pass, using the computed `indices`.
    // Indices are absolute positions in array.values() (first_offset was added
    // back when collecting them), so we can take directly from the full values.
    // Use UInt64Array for LargeList to support values arrays exceeding u32::MAX.
    let final_values = if indices.is_empty() {
        new_empty_array(&dt)
    } else if OffsetSize::IS_LARGE {
        let indices =
            UInt64Array::from(indices.into_iter().map(|i| i as u64).collect::<Vec<_>>());
        take(array.values().as_ref(), &indices, None)?