| 15 | |
| 16 | impl DenseVectorList { |
| 17 | pub fn new(path: PathBuf, elements: u64) -> io::Result<Self> { |
| 18 | let exists = path.exists(); |
| 19 | let file = OpenOptions::new() |
| 20 | .read(true) |
| 21 | .write(true) |
| 22 | .create(!exists) |
| 23 | .open(path.clone())?; |
| 24 | |
| 25 | if !exists { |
| 26 | // Set the file size, accounting for the header |
| 27 | file.set_len(elements * (QUANTIZED_VECTOR_SIZE as u64) + HEADER_SIZE as u64)?; |
| 28 | } |
| 29 | |
| 30 | let mut mmap = unsafe { MmapMut::map_mut(&file)? }; |
| 31 | |
| 32 | let used_space = if exists && file.metadata().unwrap().len() as usize > HEADER_SIZE { |
| 33 | // Read the existing used space from the file |
| 34 | let used_bytes = &mmap[0..HEADER_SIZE]; |
| 35 | u64::from_le_bytes(used_bytes.try_into().unwrap()) as usize |
| 36 | } else { |
| 37 | 0 // No data written yet, or file did not exist |
| 38 | }; |
| 39 | |
| 40 | if !exists { |
| 41 | // Initialize the header if the file is newly created |
| 42 | mmap[0..HEADER_SIZE].copy_from_slice(&(used_space as u64).to_le_bytes()); |
| 43 | } |
| 44 | |
| 45 | Ok(DenseVectorList { |
| 46 | mmap, |
| 47 | used_space, |
| 48 | path, |
| 49 | }) |
| 50 | } |
| 51 | |
| 52 | pub fn push(&mut self, vector: [u8; QUANTIZED_VECTOR_SIZE]) -> io::Result<usize> { |
| 53 | let offset = self.used_space + HEADER_SIZE; |