(&mut self)
| 82 | } |
| 83 | |
| 84 | fn consume_batch(&mut self) -> Result<ArrayRef> { |
| 85 | let next_batch_array = self.item_reader.consume_batch()?; |
| 86 | if next_batch_array.is_empty() { |
| 87 | return Ok(new_empty_array(&self.data_type)); |
| 88 | } |
| 89 | |
| 90 | let def_levels = self |
| 91 | .item_reader |
| 92 | .get_def_levels() |
| 93 | .ok_or_else(|| general_err!("item_reader def levels are None."))?; |
| 94 | |
| 95 | let rep_levels = self |
| 96 | .item_reader |
| 97 | .get_rep_levels() |
| 98 | .ok_or_else(|| general_err!("item_reader rep levels are None."))?; |
| 99 | |
| 100 | if OffsetSize::from_usize(next_batch_array.len()).is_none() { |
| 101 | return Err(general_err!( |
| 102 | "offset of {} would overflow list array", |
| 103 | next_batch_array.len() |
| 104 | )); |
| 105 | } |
| 106 | |
| 107 | if !rep_levels.is_empty() && rep_levels[0] != 0 { |
| 108 | // This implies either the source data was invalid, or the leaf column |
| 109 | // reader did not correctly delimit semantic records |
| 110 | return Err(general_err!("first repetition level of batch must be 0")); |
| 111 | } |
| 112 | |
| 113 | // A non-nullable list has a single definition level indicating if the list is empty |
| 114 | // |
| 115 | // A nullable list has two definition levels associated with it: |
| 116 | // |
| 117 | // The first identifies if the list is null |
| 118 | // The second identifies if the list is empty |
| 119 | // |
| 120 | // The child data returned above is padded with a value for each not-fully defined level. |
| 121 | // Therefore null and empty lists will correspond to a value in the child array. |
| 122 | // |
| 123 | // Whilst nulls may have a non-zero slice in the offsets array, empty lists must |
| 124 | // be of zero length. As a result we MUST filter out values corresponding to empty |
| 125 | // lists, and for consistency we do the same for nulls. |
| 126 | |
| 127 | // The output offsets for the computed ListArray |
| 128 | let mut list_offsets: Vec<OffsetSize> = Vec::with_capacity(next_batch_array.len() + 1); |
| 129 | |
| 130 | // The validity mask of the computed ListArray if nullable |
| 131 | let mut validity = self |
| 132 | .nullable |
| 133 | .then(|| BooleanBufferBuilder::new(next_batch_array.len())); |
| 134 | |
| 135 | // The offset into the filtered child data of the current level being considered |
| 136 | let mut cur_offset = 0; |
| 137 | |
| 138 | // Identifies the start of a run of values to copy from the source child data |
| 139 | let mut filter_start = None; |
| 140 | |
| 141 | // The number of child values skipped due to empty lists or nulls |
nothing calls this directly
no test coverage detected