Write binary blob: [int32 count][int32 dim] + count×dim int8.
(path: str, vectors: np.ndarray, dim: int)
| 255 | # ── Output generation ──────────────────────────────────────────────── |
| 256 | |
| 257 | def write_bin(path: str, vectors: np.ndarray, dim: int): |
| 258 | """Write binary blob: [int32 count][int32 dim] + count×dim int8.""" |
| 259 | n = vectors.shape[0] |
| 260 | # Quantize: scale to [-127, 127], round to int8 |
| 261 | quantized = np.clip(np.round(vectors * 127.0), -127, 127).astype(np.int8) |
| 262 | |
| 263 | with open(path, "wb") as f: |
| 264 | f.write(struct.pack("<ii", n, dim)) |
| 265 | f.write(quantized.tobytes()) |
| 266 | |
| 267 | size_mb = os.path.getsize(path) / (1024 * 1024) |
| 268 | print(f" {path}: {n} vectors × {dim}d = {size_mb:.1f} MB") |
| 269 | |
| 270 | |
| 271 | def write_tokens_txt(path: str, tokens: list): |