Encode raw bytes (symbol columns or string data) through a pipeline.
(raw: &[u8], codec: ColumnCodec)
| 90 | |
| 91 | /// Encode raw bytes (symbol columns or string data) through a pipeline. |
| 92 | pub fn encode_bytes_pipeline(raw: &[u8], codec: ColumnCodec) -> Result<Vec<u8>, CodecError> { |
| 93 | match codec { |
| 94 | ColumnCodec::FsstLz4 => { |
| 95 | // FSST expects an array of strings. Treat raw as a single blob. |
| 96 | let fsst_bytes = crate::fsst::encode(&[raw]); |
| 97 | Ok(crate::lz4::encode(&fsst_bytes)) |
| 98 | } |
| 99 | ColumnCodec::FsstRans => { |
| 100 | let fsst_bytes = crate::fsst::encode(&[raw]); |
| 101 | Ok(crate::rans::encode(&fsst_bytes)) |
| 102 | } |
| 103 | ColumnCodec::Raw => Ok(crate::raw::encode(raw)), |
| 104 | ColumnCodec::Lz4 => Ok(crate::lz4::encode(raw)), |
| 105 | ColumnCodec::Zstd => crate::zstd_codec::encode(raw), |
| 106 | ColumnCodec::FastLanesLz4 => { |
| 107 | // Symbol IDs are u32 — convert to i64, FastLanes pack, lz4. |
| 108 | if raw.len().is_multiple_of(4) { |
| 109 | let i64_vals: Vec<i64> = raw |
| 110 | .chunks_exact(4) |
| 111 | .map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]]) as i64) |
| 112 | .collect(); |
| 113 | encode_fastlanes_lz4_i64(&i64_vals) |
| 114 | } else { |
| 115 | Ok(crate::raw::encode(raw)) |
| 116 | } |
| 117 | } |
| 118 | _ => Ok(crate::raw::encode(raw)), |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // --------------------------------------------------------------------------- |
| 123 | // Pipeline decode |