Coordinates reading arrays based on data types. `variadic_counts` encodes the number of buffers to read for variadic types (e.g., Utf8View, BinaryView) When encounter such types, we pop from the front of the queue to get the number of buffers to read. Notes: In the IPC format, null buffers are always set, but may be empty. We discard them if an array has 0 nulls Numeric values inside list arrays
(
&mut self,
field: &Field,
variadic_counts: &mut VecDeque<i64>,
)
| 86 | /// - read the buffer as 64-bit (signed integer or float), and |
| 87 | /// - cast the 64-bit array to the appropriate data type |
| 88 | fn create_array( |
| 89 | &mut self, |
| 90 | field: &Field, |
| 91 | variadic_counts: &mut VecDeque<i64>, |
| 92 | ) -> Result<ArrayRef, ArrowError> { |
| 93 | let data_type = field.data_type(); |
| 94 | match data_type { |
| 95 | Utf8 | Binary | LargeBinary | LargeUtf8 => { |
| 96 | let field_node = self.next_node(field)?; |
| 97 | let buffers = [ |
| 98 | self.next_buffer()?, |
| 99 | self.next_buffer()?, |
| 100 | self.next_buffer()?, |
| 101 | ]; |
| 102 | self.create_primitive_array(field_node, data_type, &buffers) |
| 103 | } |
| 104 | BinaryView | Utf8View => { |
| 105 | let count = variadic_counts |
| 106 | .pop_front() |
| 107 | .ok_or(ArrowError::IpcError(format!( |
| 108 | "Missing variadic count for {data_type} column" |
| 109 | )))?; |
| 110 | let count = count + 2; // view and null buffer. |
| 111 | let buffers = (0..count) |
| 112 | .map(|_| self.next_buffer()) |
| 113 | .collect::<Result<Vec<_>, _>>()?; |
| 114 | let field_node = self.next_node(field)?; |
| 115 | self.create_primitive_array(field_node, data_type, &buffers) |
| 116 | } |
| 117 | FixedSizeBinary(_) => { |
| 118 | let field_node = self.next_node(field)?; |
| 119 | let buffers = [self.next_buffer()?, self.next_buffer()?]; |
| 120 | self.create_primitive_array(field_node, data_type, &buffers) |
| 121 | } |
| 122 | List(list_field) | LargeList(list_field) | Map(list_field, _) => { |
| 123 | let list_node = self.next_node(field)?; |
| 124 | let list_buffers = [self.next_buffer()?, self.next_buffer()?]; |
| 125 | let values = self.create_array(list_field, variadic_counts)?; |
| 126 | self.create_list_array(list_node, data_type, &list_buffers, values) |
| 127 | } |
| 128 | ListView(list_field) | LargeListView(list_field) => { |
| 129 | let list_node = self.next_node(field)?; |
| 130 | let list_buffers = [ |
| 131 | self.next_buffer()?, // null buffer |
| 132 | self.next_buffer()?, // offsets |
| 133 | self.next_buffer()?, // sizes |
| 134 | ]; |
| 135 | let values = self.create_array(list_field, variadic_counts)?; |
| 136 | self.create_list_view_array(list_node, data_type, &list_buffers, values) |
| 137 | } |
| 138 | FixedSizeList(list_field, _) => { |
| 139 | let list_node = self.next_node(field)?; |
| 140 | let list_buffers = [self.next_buffer()?]; |
| 141 | let values = self.create_array(list_field, variadic_counts)?; |
| 142 | self.create_list_array(list_node, data_type, &list_buffers, values) |
| 143 | } |
| 144 | Struct(struct_fields) => { |
| 145 | let struct_node = self.next_node(field)?; |
no test coverage detected