Function encode

nodedb-codec/src/rans.rs:45–124

Compress bytes using interleaved rANS.

(data: &[u8])

/// Compress bytes using interleaved rANS.
pub fn encode(data: &[u8]) -> Vec<u8> {
    if data.is_empty() {
        let out = vec![0u8; HEADER_SIZE];
        // uncompressed_size = 0, freq table = all zeros, compressed_size = 0
        return out;
    }

    // Build frequency table.
    let mut freqs = [0u32; 256];
    for &b in data {
        freqs[b as usize] += 1;
    }

    // Normalize frequencies to sum to PROB_SCALE.
    let norm_freqs = normalize_frequencies(&freqs, data.len());

    // Build cumulative frequency table.
    let (cum_freqs, sym_freqs) = build_cum_table(&norm_freqs);

    // Encode using 4 interleaved streams.
    // Each stream processes every 4th byte: stream 0 gets bytes 0,4,8,...
    let mut streams: [Vec<u8>; NUM_STREAMS] = std::array::from_fn(|_| Vec::new());
    let mut states = [RANS_L; NUM_STREAMS];

    // Encode in REVERSE order (rANS encodes backward, decodes forward).
    for i in (0..data.len()).rev() {
        let stream_idx = i % NUM_STREAMS;
        let sym = data[i] as usize;
        let freq = sym_freqs[sym];
        let start = cum_freqs[sym];

        if freq == 0 {
            continue; // Symbol with zero frequency — shouldn't happen after normalization.
        }

        rans_encode_symbol(
            &mut states[stream_idx],
            &mut streams[stream_idx],
            start,
            freq,
        );
    }

    // Flush final states.
    for i in 0..NUM_STREAMS {
        let s = states[i];
        streams[i].push((s & 0xFF) as u8);
        streams[i].push(((s >> 8) & 0xFF) as u8);
        streams[i].push(((s >> 16) & 0xFF) as u8);
        streams[i].push(((s >> 24) & 0xFF) as u8);
    }

    // Build output.
    let total_compressed: usize = streams.iter().map(|s| s.len()).sum();
    let mut out = Vec::with_capacity(HEADER_SIZE + total_compressed + NUM_STREAMS * 4);

    // Header: uncompressed size.
    out.extend_from_slice(&(data.len() as u32).to_le_bytes());