Function decode_binary_view_inner

arrow-row/src/variable.rs:312–394

(
    rows: &mut [&[u8]],
    options: SortOptions,
    validate_utf8: bool,
)

}

fn decode_binary_view_inner(
    rows: &mut [&[u8]],
    options: SortOptions,
    validate_utf8: bool,
) -> BinaryViewArray {
    let len = rows.len();
    let inline_str_max_len = MAX_INLINE_VIEW_LEN as usize;

    let mut null_count = 0;

    let nulls = MutableBuffer::collect_bool(len, |x| {
        let valid = rows[x][0] != null_sentinel(options);
        null_count += !valid as usize;
        valid
    });

    // If we are validating UTF-8, decode all string values (including short strings)
    // into the values buffer and validate UTF-8 once. If not validating,
    // we save memory by only copying long strings to the values buffer, as short strings
    // will be inlined into the view and do not need to be stored redundantly.
    let values_capacity = if validate_utf8 {
        // Capacity for all long and short strings
        rows.iter().map(|row| decoded_len(row, options)).sum()
    } else {
        // Capacity for all long strings plus room for one short string
        rows.iter().fold(0, |acc, row| {
            let len = decoded_len(row, options);
            if len > inline_str_max_len {
                acc + len
            } else {
                acc
            }
        }) + inline_str_max_len
    };
    let mut values = MutableBuffer::new(values_capacity);

    let mut views = BufferBuilder::<u128>::new(len);
    for row in rows {
        let start_offset = values.len();
        let offset = decode_blocks(row, options, |b| values.extend_from_slice(b));
        // Measure string length via change in values buffer.
        // Used to check if decoded value should be truncated (short string) when validate_utf8 is false
        let decoded_len = values.len() - start_offset;
        if row[0] == null_sentinel(options) {
            debug_assert_eq!(offset, 1);
            debug_assert_eq!(start_offset, values.len());
            views.append(0);
        } else {
            // Safety: we just appended the data to the end of the buffer
            let val = unsafe { values.get_unchecked_mut(start_offset..) };

            if options.descending {
                val.iter_mut().for_each(|o| *o = !*o);
            }

            let view = make_view(val, 0, start_offset as u32);
            views.append(view);