Method try_append_value

arrow-array/src/builder/generic_bytes_view_builder.rs:339–422

(&mut self, value: impl AsRef<T::Native>)

    /// - String length exceeds `u32::MAX`
    #[inline]
    pub fn try_append_value(&mut self, value: impl AsRef<T::Native>) -> Result<(), ArrowError> {
        let v: &[u8] = value.as_ref().as_ref();
        let length: u32 = v.len().try_into().map_err(|_| {
            ArrowError::InvalidArgumentError(format!("String length {} exceeds u32::MAX", v.len()))
        })?;

        if length <= MAX_INLINE_VIEW_LEN {
            let mut view_buffer = [0; 16];
            view_buffer[0..4].copy_from_slice(&length.to_le_bytes());
            view_buffer[4..4 + v.len()].copy_from_slice(v);
            self.views_buffer.push(u128::from_le_bytes(view_buffer));
            self.null_buffer_builder.append_non_null();
            return Ok(());
        }

        // Deduplication if:
        // (1) deduplication is enabled.
        // (2) len > `MAX_INLINE_VIEW_LEN` and len <= `max_deduplication_len`
        let can_deduplicate = self.string_tracker.is_some()
            && self
                .max_deduplication_len
                .map(|max_length| length <= max_length)
                .unwrap_or(true);
        if can_deduplicate {
            if let Some((mut ht, hasher)) = self.string_tracker.take() {
                let hash_val = hasher.hash_one(v);
                let hasher_fn = |v: &_| hasher.hash_one(v);

                let entry = ht.entry(
                    hash_val,
                    |idx| {
                        let stored_value = self.get_value(*idx);
                        v == stored_value
                    },
                    hasher_fn,
                );
                match entry {
                    Entry::Occupied(occupied) => {
                        // If the string already exists, we will directly use the view
                        let idx = occupied.get();
                        self.views_buffer.push(self.views_buffer[*idx]);
                        self.null_buffer_builder.append_non_null();
                        self.string_tracker = Some((ht, hasher));
                        return Ok(());
                    }
                    Entry::Vacant(vacant) => {
                        // o.w. we insert the (string hash -> view index)
                        // the idx is current length of views_builder, as we are inserting a new view
                        vacant.insert(self.views_buffer.len());
                    }
                }
                self.string_tracker = Some((ht, hasher));
            }
        }

        let required_cap = self.in_progress.len() + v.len();
        if self.in_progress.capacity() < required_cap {
            self.flush_in_progress();