Function with_hashes

datafusion/common/src/hash_utils.rs:100–141

Creates hashes for the given arrays using a thread-local buffer, then calls the provided callback with an immutable reference to the computed hashes. This function manages a thread-local buffer to avoid repeated allocations. The buffer is automatically truncated if it exceeds `MAX_BUFFER_SIZE` after use. # Arguments `arrays` - The arrays to hash (must contain at least one array) `random_state` -

(
    arrays: I,
    random_state: &RandomState,
    callback: F,
)

/// })?;
/// ```
pub fn with_hashes<I, T, F, R>(
    arrays: I,
    random_state: &RandomState,
    callback: F,
) -> Result<R>
where
    I: IntoIterator<Item = T>,
    T: AsDynArray,
    F: FnOnce(&[u64]) -> Result<R>,
{
    // Peek at the first array to determine buffer size without fully collecting
    let mut iter = arrays.into_iter().peekable();

    // Get the required size from the first array
    let required_size = match iter.peek() {
        Some(arr) => arr.as_dyn_array().len(),
        None => return _internal_err!("with_hashes requires at least one array"),
    };

    HASH_BUFFER.try_with(|cell| {
        let mut buffer = cell.try_borrow_mut()
            .map_err(|_| _internal_datafusion_err!("with_hashes cannot be called reentrantly on the same thread"))?;

        // Ensure buffer has sufficient length, clearing old values
        buffer.clear();
        buffer.resize(required_size, 0);

        // Create hashes in the buffer - this consumes the iterator
        create_hashes(iter, random_state, &mut buffer[..required_size])?;

        // Execute the callback with an immutable slice
        let result = callback(&buffer[..required_size])?;

        // Cleanup: truncate if buffer grew too large
        if buffer.capacity() > MAX_BUFFER_SIZE {
            buffer.truncate(MAX_BUFFER_SIZE);
            buffer.shrink_to_fit();
        }

        Ok(result)
    }).map_err(|_| _internal_datafusion_err!("with_hashes cannot access thread-local storage during or after thread destruction"))?
}

#[cfg(not(feature = "force_hash_collisions"))]
fn hash_null(random_state: &RandomState, hashes_buffer: &'_ mut [u64], mul_col: bool) {