Function encode

nodedb-codec/src/fsst.rs:156–193

Encode a batch of strings using FSST compression. Trains a symbol table on the input, then encodes each string as a sequence of symbol indices and escaped literals.

(strings: &[&[u8]])

/// Trains a symbol table on the input, then encodes each string as a
/// sequence of symbol indices and escaped literals.
pub fn encode(strings: &[&[u8]]) -> Vec<u8> {
    let table = SymbolTable::train(strings);

    // Encode each string.
    let mut encoded_strings: Vec<Vec<u8>> = Vec::with_capacity(strings.len());
    for s in strings {
        encoded_strings.push(encode_string(&table, s));
    }

    // Build wire format.
    let mut out = Vec::new();

    // Symbol table.
    out.extend_from_slice(&(table.symbol_count() as u16).to_le_bytes());
    for sym in &table.symbols {
        out.push(sym.len() as u8);
        out.extend_from_slice(sym);
    }

    // Encoded strings with offset table.
    let total_encoded: usize = encoded_strings.iter().map(|s| s.len()).sum();
    out.extend_from_slice(&(total_encoded as u32).to_le_bytes());
    out.extend_from_slice(&(strings.len() as u32).to_le_bytes());

    // Cumulative offsets.
    let mut offset = 0u32;
    for es in &encoded_strings {
        offset += es.len() as u32;
        out.extend_from_slice(&offset.to_le_bytes());
    }

    // Encoded data.
    for es in &encoded_strings {
        out.extend_from_slice(es);
    }

    out
}

/// Decode FSST-compressed data back to strings.
pub fn decode(data: &[u8]) -> Result<Vec<Vec<u8>>, CodecError> {