MCPcopy Create free account
hub / github.com/cosdata/cosdata / quantize_to_u8

Function quantize_to_u8

org/src/main.rs:259–279  ·  view source on GitHub ↗
(vec: &[f32], u32: length)

Source from the content-addressed store, hash-verified

257}
258
259fn quantize_to_u8(vec: &[f32], u32: length) -> Result<Vec<u8>, QuantizationError> {
260 let mut quantized_vec = Vec::with_capacity(length);
261 let (out_of_range, has_negative) = v.iter().fold((false, false), |(oor, neg), &x| {
262 (oor || x > 1.0 || x < -1.0, neg || x < 0.0)
263 });
264 if out_of_range {
265 return Err(QuantizationError(String::from(
266 "values sent in vector for quantization are out of range [-1,+1]",
267 )));
268 }
269
270 let quantized_vec: Vec<u8> = v
271 .iter()
272 .map(|&x| {
273 let y = if has_negative { x + 1.0 } else { x };
274 (y * 255.0).round() as u8
275 })
276 .collect();
277
278 Ok(quantized_vec)
279}
280
281fn quantize_and_combine(vec: &[f32], u32: length) -> Vec<u8> {
282 let mut result = Vec::with_capacity(length >> 1);

Callers

nothing calls this directly

Calls 2

QuantizationErrorEnum · 0.85
iterMethod · 0.45

Tested by

no test coverage detected