Generic version of [`Self::insert_if_new`] that handles `ByteArrayType` (both String and Binary) Note this is the only function that is generic on [`ByteArrayType`], which avoids having to template the entire structure, making the code simpler and understand and reducing code bloat due to duplication. See comments on `insert_if_new` for more details
(
&mut self,
values: &ArrayRef,
mut make_payload_fn: MP,
mut observe_payload_fn: OP,
)
| 336 | /// |
| 337 | /// See comments on `insert_if_new` for more details |
| 338 | fn insert_if_new_inner<MP, OP, B>( |
| 339 | &mut self, |
| 340 | values: &ArrayRef, |
| 341 | mut make_payload_fn: MP, |
| 342 | mut observe_payload_fn: OP, |
| 343 | ) where |
| 344 | MP: FnMut(Option<&[u8]>) -> V, |
| 345 | OP: FnMut(V), |
| 346 | B: ByteArrayType, |
| 347 | { |
| 348 | // step 1: compute hashes |
| 349 | let batch_hashes = &mut self.hashes_buffer; |
| 350 | batch_hashes.clear(); |
| 351 | batch_hashes.resize(values.len(), 0); |
| 352 | create_hashes([values], &self.random_state, batch_hashes) |
| 353 | // hash is supported for all types and create_hashes only |
| 354 | // returns errors for unsupported types |
| 355 | .unwrap(); |
| 356 | |
| 357 | // step 2: insert each value into the set, if not already present |
| 358 | let values = values.as_bytes::<B>(); |
| 359 | |
| 360 | // Ensure lengths are equivalent |
| 361 | assert_eq!(values.len(), batch_hashes.len()); |
| 362 | |
| 363 | for (value, &hash) in values.iter().zip(batch_hashes.iter()) { |
| 364 | // handle null value |
| 365 | let Some(value) = value else { |
| 366 | let payload = if let Some(&(payload, _offset)) = self.null.as_ref() { |
| 367 | payload |
| 368 | } else { |
| 369 | let payload = make_payload_fn(None); |
| 370 | let null_index = self.offsets.len() - 1; |
| 371 | // nulls need a zero length in the offset buffer |
| 372 | let offset = self.buffer.len(); |
| 373 | self.offsets.push(O::usize_as(offset)); |
| 374 | self.null = Some((payload, null_index)); |
| 375 | payload |
| 376 | }; |
| 377 | observe_payload_fn(payload); |
| 378 | continue; |
| 379 | }; |
| 380 | |
| 381 | // get the value as bytes |
| 382 | let value: &[u8] = value.as_ref(); |
| 383 | let value_len = O::usize_as(value.len()); |
| 384 | |
| 385 | // value is "small" |
| 386 | let payload = if value.len() <= SHORT_VALUE_LEN { |
| 387 | let inline = value.iter().fold(0usize, |acc, &x| (acc << 8) | x as usize); |
| 388 | |
| 389 | // is value is already present in the set? |
| 390 | let entry = self.map.find_mut(hash, |header| { |
| 391 | // compare value if hashes match |
| 392 | if header.hash != hash || header.len != value_len { |
| 393 | return false; |
| 394 | } |
| 395 | // value is stored inline so no need to consult buffer |
nothing calls this directly
no test coverage detected