Function encode

rust/dialog-encoding/src/buffer.rs:99–147

Encodes a columnar data structure into a binary buffer with deduplication. This function takes a data structure that implements [`Cellular`] and encodes it into a compact binary format that enables zero-copy reading. Identical byte sequences are automatically deduplicated to minimize storage space. # Arguments `layout` - The data structure to encode, must implement [`Cellular`] `buffer` - The o

(
    layout: &'a Layout,
    mut buffer: Buffer,
)

/// // The encoded buffer will deduplicate the repeated "hello"
/// ```
pub fn encode<'a, Layout, Buffer>(
    layout: &'a Layout,
    mut buffer: Buffer,
) -> Result<(), std::io::Error>
where
    Layout: Cellular<'a>,
    Buffer: Write,
{
    let mut data = Cursor::new(Vec::new());
    let mut ranges = Cursor::new(Vec::new());
    let mut cells = Cursor::new(Vec::new());
    let mut bytes_to_index = BTreeMap::<&'a [u8], u64>::new();
    let mut next_index = 0u64;
    let mut data_length = 0usize;

    for cell in layout.cells() {
        if let Some(index) = bytes_to_index.get(cell) {
            leb128::write::unsigned(&mut cells, *index)?;
        } else {
            leb128::write::unsigned(&mut ranges, data_length as u64)?;
            leb128::write::unsigned(&mut ranges, cell.len() as u64)?;

            data_length += cell.len();
            data.write_all(cell)?;
            bytes_to_index.insert(cell, next_index);

            leb128::write::unsigned(&mut cells, next_index)?;

            next_index += 1;
        }
    }

    let data = data.into_inner();
    let ranges = ranges.into_inner();
    let cells = cells.into_inner();

    // [ data length ][ data ]
    leb128::write::unsigned(&mut buffer, data.len() as u64)?;
    buffer.write_all(&data)?;

    // [ ranges length ][ ranges ]
    leb128::write::unsigned(&mut buffer, ranges.len() as u64)?;
    buffer.write_all(&ranges)?;

    // [ cells ]
    buffer.write_all(&cells)?;

    Ok(())
}

/// Decodes a binary buffer into a columnar data structure with zero-copy access.
///