Method read

parquet/src/arrow/array_reader/byte_array_dictionary.rs:317–375

(&mut self, out: &mut Self::Buffer, num_values: usize)

    }

    fn read(&mut self, out: &mut Self::Buffer, num_values: usize) -> Result<usize> {
        match self.decoder.as_mut().expect("decoder set") {
            MaybeDictionaryDecoder::Fallback(decoder) => {
                decoder.read(out.spill_values()?, num_values, None)
            }
            MaybeDictionaryDecoder::Dict {
                decoder,
                max_remaining_values,
            } => {
                let len = num_values.min(*max_remaining_values);

                let dict = self
                    .dict
                    .as_ref()
                    .ok_or_else(|| general_err!("missing dictionary page for column"))?;

                assert_eq!(dict.data_type(), &self.value_type);

                if dict.is_empty() {
                    return Ok(0); // All data must be NULL
                }

                match out.as_keys(dict) {
                    Some(keys) => {
                        // Happy path - can just copy keys
                        // Keys will be validated on conversion to arrow

                        // TODO: Push vec into decoder (#5177)
                        let start = keys.len();
                        keys.resize(start + len, K::default());
                        let len = decoder.get_batch(&mut keys[start..])?;
                        keys.truncate(start + len);
                        *max_remaining_values -= len;
                        Ok(len)
                    }
                    None => {
                        // Sad path - need to recompute dictionary
                        //
                        // This either means we crossed into a new column chunk whilst
                        // reading this batch, or encountered non-dictionary encoded data
                        let values = out.spill_values()?;
                        let mut keys = vec![K::default(); len];
                        let len = decoder.get_batch(&mut keys)?;

                        assert_eq!(dict.data_type(), &self.value_type);

                        let data = dict.to_data();
                        let dict_buffers = data.buffers();
                        let dict_offsets = dict_buffers[0].typed_data::<V>();
                        let dict_values = dict_buffers[1].as_slice();

                        values.extend_from_dictionary(&keys[..len], dict_offsets, dict_values)?;
                        *max_remaining_values -= len;
                        Ok(len)
                    }
                }
            }
        }