Method read_impl

parquet/src/arrow/array_reader/byte_view_array.rs:328–433

(
        &mut self,
        output: &mut ViewBuffer,
        len: usize,
    )

    }

    fn read_impl<const VALIDATE_UTF8: bool>(
        &mut self,
        output: &mut ViewBuffer,
        len: usize,
    ) -> Result<usize> {
        // avoid creating a new buffer if the last buffer is the same as the current buffer
        // This is especially useful when row-level filtering is applied, where we call lots of small `read` over the same buffer.
        let block_id = {
            if output.buffers.last().is_some_and(|x| x.ptr_eq(&self.buf)) {
                output.buffers.len() as u32 - 1
            } else {
                output.append_block(self.buf.clone())
            }
        };

        let to_read = len.min(self.max_remaining_values);

        let buf: &[u8] = self.buf.as_ref();
        let buf_len = buf.len();
        let mut end_offset = self.offset;
        let mut utf8_validation_begin = end_offset;

        output.views.reserve(to_read);

        // Safety: we reserved enough space in output.views
        // and we will only write up to to_read views / track how many views we wrote.
        // Ideally, we would use `Vec::extend` here, but this generates sub-optimal code.
        let views_ptr = output.views.as_mut_ptr().wrapping_add(output.views.len());
        for i in 0..to_read {
            let start_offset = end_offset + 4;

            if start_offset > buf_len {
                return Err(ParquetError::EOF("eof decoding byte array".into()));
            }

            // Safety: we have checked that start_offset <= buf_len
            let len = u32::from_le_bytes(
                unsafe { buf.get_unchecked(end_offset..start_offset) }
                    .try_into()
                    .unwrap(),
            );

            end_offset = start_offset + len as usize;

            if end_offset > buf_len {
                return Err(ParquetError::EOF("eof decoding byte array".into()));
            }

            if VALIDATE_UTF8 {
                // It seems you are trying to understand what's going on here, take a breath and be patient.
                // Utf-8 validation is a non-trivial task, here are some background facts:
                // (1) Validating one 2048-byte string is much faster than validating 128 of 16-byte string.
                //     As shown in https://github.com/apache/arrow-rs/pull/6009#issuecomment-2211174229
                //     Potentially because the SIMD operations favor longer strings.
                // (2) Practical strings are short, 99% of strings are smaller than 100 bytes, as shown in paper:
                //     https://www.vldb.org/pvldb/vol17/p148-zeng.pdf, Figure 5f.
                // (3) Parquet plain encoding makes utf-8 validation harder,
                //     because it stores the length of each string right before the string.