Parse an fvecs file and return a list of float32 vectors. Each vector is stored as: 4 bytes (int32 dimension) + dim * 4 bytes (float32 values) Args: filepath: Path to the .fvecs file. Returns: List of lists of floats. Raises: FileNotFoundError: If
(filepath)
| 21 | |
| 22 | |
| 23 | def read_fvecs(filepath): |
| 24 | """Parse an fvecs file and return a list of float32 vectors. |
| 25 | |
| 26 | Each vector is stored as: |
| 27 | 4 bytes (int32 dimension) + dim * 4 bytes (float32 values) |
| 28 | |
| 29 | Args: |
| 30 | filepath: Path to the .fvecs file. |
| 31 | |
| 32 | Returns: |
| 33 | List of lists of floats. |
| 34 | |
| 35 | Raises: |
| 36 | FileNotFoundError: If the file does not exist. |
| 37 | ValueError: If the file format is invalid or dimensions are inconsistent. |
| 38 | """ |
| 39 | if not os.path.isfile(filepath): |
| 40 | raise FileNotFoundError(f"fvecs file not found: {filepath}") |
| 41 | |
| 42 | vectors = [] |
| 43 | expected_dim = None |
| 44 | file_size = os.path.getsize(filepath) |
| 45 | |
| 46 | with open(filepath, "rb") as f: |
| 47 | while f.tell() < file_size: |
| 48 | # Read dimension (4 bytes, int32) |
| 49 | dim_bytes = f.read(4) |
| 50 | if len(dim_bytes) < 4: |
| 51 | if len(dim_bytes) == 0: |
| 52 | break # Clean EOF |
| 53 | raise ValueError( |
| 54 | f"Unexpected end of file reading dimension at byte {f.tell() - len(dim_bytes)} " |
| 55 | f"in {filepath}" |
| 56 | ) |
| 57 | |
| 58 | (dim,) = struct.unpack("<i", dim_bytes) |
| 59 | if dim <= 0: |
| 60 | raise ValueError( |
| 61 | f"Invalid dimension {dim} at vector index {len(vectors)} in {filepath}" |
| 62 | ) |
| 63 | |
| 64 | if expected_dim is None: |
| 65 | expected_dim = dim |
| 66 | elif dim != expected_dim: |
| 67 | raise ValueError( |
| 68 | f"Dimension mismatch at vector index {len(vectors)}: " |
| 69 | f"expected {expected_dim}, got {dim} in {filepath}" |
| 70 | ) |
| 71 | |
| 72 | # Read vector data (dim * 4 bytes, float32) |
| 73 | vec_bytes = f.read(dim * 4) |
| 74 | if len(vec_bytes) < dim * 4: |
| 75 | raise ValueError( |
| 76 | f"Unexpected end of file reading vector data at vector index {len(vectors)} " |
| 77 | f"in {filepath}. Expected {dim * 4} bytes, got {len(vec_bytes)}" |
| 78 | ) |
| 79 | |
| 80 | values = list(struct.unpack(f"<{dim}f", vec_bytes)) |
no outgoing calls