Method build_sq8_for_index

nodedb-vector/src/collection/quantize.rs:52–83

Build SQ8 quantized data for an HNSW index. Returns `None` when there are too few live vectors for stable min/max calibration.

(index: &HnswIndex)

    /// Returns `None` when there are too few live vectors for stable
    /// min/max calibration.
    pub fn build_sq8_for_index(index: &HnswIndex) -> Option<(Sq8Codec, Vec<u8>)> {
        if index.live_count() < 1000 {
            return None;
        }
        let dim = index.dim();
        let n = index.len();

        let mut refs: Vec<&[f32]> = Vec::with_capacity(n);
        for i in 0..n {
            if !index.is_deleted(i as u32)
                && let Some(v) = index.get_vector(i as u32)
            {
                refs.push(v);
            }
        }
        if refs.is_empty() {
            return None;
        }

        let codec = Sq8Codec::calibrate(&refs, dim);

        let mut data = Vec::with_capacity(dim * n);
        for i in 0..n {
            if let Some(v) = index.get_vector(i as u32) {
                data.extend(codec.quantize(v));
            } else {
                data.extend(vec![0u8; dim]);
            }
        }

        Some((codec, data))
    }

    /// Train a PQ codec from a built HNSW index's live vectors.
    pub fn build_pq_for_index(index: &HnswIndex, pq_m: usize) -> Option<(PqCodec, Vec<u8>)> {