Function chunk_content

atomic-core/src/change/format_v3/chunking.rs:481–515

Split content into variable-size chunks using FastCDC. Returns a list of [`ContentChunk`] descriptors. Each chunk's data can be accessed via `&data[chunk.offset..chunk.offset + chunk.length]` or via [`ContentChunk::data`]. # Small Data Handling If the input data is smaller than `options.min_size`, a single chunk is returned covering the entire input. If the input is empty, an empty vector is re

(data: &[u8], options: &ChunkingOptions)

/// }
/// ```
pub fn chunk_content(data: &[u8], options: &ChunkingOptions) -> Vec<ContentChunk> {
    if data.is_empty() {
        return Vec::new();
    }

    // For data smaller than min_size, return a single chunk
    if data.len() <= options.min_size {
        return vec![ContentChunk {
            offset: 0,
            length: data.len(),
            hash: *blake3::hash(data).as_bytes(),
            index: 0,
        }];
    }

    let chunker = fastcdc::v2020::FastCDC::new(
        data,
        options.min_size as u32,
        options.avg_size as u32,
        options.max_size as u32,
    );

    chunker
        .enumerate()
        .map(|(i, chunk)| {
            let chunk_data = &data[chunk.offset..chunk.offset + chunk.length];
            ContentChunk {
                offset: chunk.offset,
                length: chunk.length,
                hash: *blake3::hash(chunk_data).as_bytes(),
                index: i as u32,
            }
        })
        .collect()
}

/// Split content into chunks and return chunking statistics.
///