Returns the length, in bytes, of the buffer `i` (indexed according to the C data interface) Rust implementation uses fixed-sized buffers, which require knowledge of their `len`. for variable-sized buffers, such as the second buffer of a stringArray, we need to fetch offset buffer's len to build the second buffer.
(
&self,
i: usize,
variadic_buffer_lengths: &[i64],
dt: &DataType,
)
| 468 | /// for variable-sized buffers, such as the second buffer of a stringArray, we need |
| 469 | /// to fetch offset buffer's len to build the second buffer. |
| 470 | fn buffer_len( |
| 471 | &self, |
| 472 | i: usize, |
| 473 | variadic_buffer_lengths: &[i64], |
| 474 | dt: &DataType, |
| 475 | ) -> Result<usize> { |
| 476 | // Special handling for dictionary type as we only care about the key type in the case. |
| 477 | let data_type = match dt { |
| 478 | DataType::Dictionary(key_data_type, _) => key_data_type.as_ref(), |
| 479 | dt => dt, |
| 480 | }; |
| 481 | |
| 482 | // `ffi::ArrowArray` records array offset, we need to add it back to the |
| 483 | // buffer length to get the actual buffer length. |
| 484 | let length = self.array.len() + self.array.offset(); |
| 485 | |
| 486 | // Inner type is not important for buffer length. |
| 487 | Ok(match (&data_type, i) { |
| 488 | (DataType::Utf8, 1) |
| 489 | | (DataType::LargeUtf8, 1) |
| 490 | | (DataType::Binary, 1) |
| 491 | | (DataType::LargeBinary, 1) |
| 492 | | (DataType::List(_), 1) |
| 493 | | (DataType::LargeList(_), 1) |
| 494 | | (DataType::Map(_, _), 1) => { |
| 495 | // the len of the offset buffer (buffer 1) equals length + 1 |
| 496 | let bits = bit_width(data_type, i)?; |
| 497 | debug_assert_eq!(bits % 8, 0); |
| 498 | (length + 1) * (bits / 8) |
| 499 | } |
| 500 | (DataType::ListView(_), 1) |
| 501 | | (DataType::ListView(_), 2) |
| 502 | | (DataType::LargeListView(_), 1) |
| 503 | | (DataType::LargeListView(_), 2) => { |
| 504 | let bits = bit_width(data_type, i)?; |
| 505 | debug_assert_eq!(bits % 8, 0); |
| 506 | length * (bits / 8) |
| 507 | } |
| 508 | (DataType::Utf8, 2) | (DataType::Binary, 2) => { |
| 509 | if self.array.is_empty() { |
| 510 | return Ok(0); |
| 511 | } |
| 512 | |
| 513 | // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) |
| 514 | let len = self.buffer_len(1, variadic_buffer_lengths, dt)?; |
| 515 | // first buffer is the null buffer => add(1) |
| 516 | // we assume that pointer is aligned for `i32`, as Utf8 uses `i32` offsets. |
| 517 | #[allow(clippy::cast_ptr_alignment)] |
| 518 | let offset_buffer = self.array.buffer(1) as *const i32; |
| 519 | // get last offset |
| 520 | (unsafe { *offset_buffer.add(len / size_of::<i32>() - 1) }) as usize |
| 521 | } |
| 522 | (DataType::LargeUtf8, 2) | (DataType::LargeBinary, 2) => { |
| 523 | if self.array.is_empty() { |
| 524 | return Ok(0); |
| 525 | } |
| 526 | |
| 527 | // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) |