Computes the logical nulls for a sparse union, optimized for when there's a lot of fields fully null
(&self, mut nulls: Vec<(i8, NullBuffer)>)
| 459 | |
| 460 | /// Computes the logical nulls for a sparse union, optimized for when there's a lot of fields fully null |
| 461 | fn mask_sparse_skip_fully_null(&self, mut nulls: Vec<(i8, NullBuffer)>) -> BooleanBuffer { |
| 462 | let fields = match self.data_type() { |
| 463 | DataType::Union(fields, _) => fields, |
| 464 | _ => unreachable!("Union array's data type is not a union!"), |
| 465 | }; |
| 466 | |
| 467 | let type_ids = fields.iter().map(|(id, _)| id).collect::<HashSet<_>>(); |
| 468 | let with_nulls = nulls.iter().map(|(id, _)| *id).collect::<HashSet<_>>(); |
| 469 | |
| 470 | let without_nulls_ids = type_ids |
| 471 | .difference(&with_nulls) |
| 472 | .copied() |
| 473 | .collect::<Vec<_>>(); |
| 474 | |
| 475 | nulls.retain(|(_, nulls)| nulls.null_count() < nulls.len()); |
| 476 | |
| 477 | // Example logic for a union with 6 fields, a, b & c with nulls, d & e without nulls, and f fully_null: |
| 478 | // let [a_nulls, b_nulls, c_nulls] = nulls; |
| 479 | // let [is_a, is_b, is_c, is_d, is_e] = masks; |
| 480 | // let union_chunk_nulls = is_d | is_e | (is_a & a_nulls) | (is_b & b_nulls) | (is_c & c_nulls) |
| 481 | self.mask_sparse_helper( |
| 482 | nulls, |
| 483 | |type_ids_chunk_array, nulls_masks_iters| { |
| 484 | let union_nulls = nulls_masks_iters.iter_mut().fold( |
| 485 | 0, |
| 486 | |union_nulls, (field_type_id, nulls_iter)| { |
| 487 | let field_nulls = nulls_iter.next().unwrap(); |
| 488 | |
| 489 | if field_nulls == 0 { |
| 490 | union_nulls |
| 491 | } else { |
| 492 | let is_field = selection_mask(type_ids_chunk_array, *field_type_id); |
| 493 | |
| 494 | union_nulls | (is_field & field_nulls) |
| 495 | } |
| 496 | }, |
| 497 | ); |
| 498 | |
| 499 | // Given the example above, this is the is_d_or_e = (is_d | is_e) part |
| 500 | let without_nulls_selected = |
| 501 | without_nulls_selected(type_ids_chunk_array, &without_nulls_ids); |
| 502 | |
| 503 | // if a field without nulls is selected, the value is always true(set bit) |
| 504 | // otherwise, the true/set bits have been computed above |
| 505 | union_nulls | without_nulls_selected |
| 506 | }, |
| 507 | |type_ids_remainder, bit_chunks| { |
| 508 | let union_nulls = |
| 509 | bit_chunks |
| 510 | .iter() |
| 511 | .fold(0, |union_nulls, (field_type_id, field_bit_chunks)| { |
| 512 | let is_field = selection_mask(type_ids_remainder, *field_type_id); |
| 513 | let field_nulls = field_bit_chunks.remainder_bits(); |
| 514 | |
| 515 | union_nulls | is_field & field_nulls |
| 516 | }); |
| 517 | |
| 518 | union_nulls | without_nulls_selected(type_ids_remainder, &without_nulls_ids) |
no test coverage detected