| 454 | |
| 455 | |
| 456 | class UnquantizedTensor(Tensor): |
| 457 | def __init__(self, ndarray: NDArray) -> None: |
| 458 | assert isinstance(ndarray, np.ndarray) |
| 459 | self.ndarray = ndarray |
| 460 | self.data_type = NUMPY_TYPE_TO_DATA_TYPE[ndarray.dtype] |
| 461 | |
| 462 | def astype(self, data_type: DataType) -> Tensor: |
| 463 | dtype = data_type.dtype |
| 464 | if self.data_type == DT_BF16: |
| 465 | self.ndarray = bf16_to_fp32(self.ndarray) |
| 466 | return UnquantizedTensor(self.ndarray.astype(dtype)) |
| 467 | |
| 468 | def to_ggml(self) -> UnquantizedTensor: |
| 469 | return self |
| 470 | |
| 471 | def permute_part(self, n_part: int, n_head: int, n_head_kv: int) -> UnquantizedTensor: |
| 472 | r = self.ndarray.shape[0] // 3 |
| 473 | return UnquantizedTensor(permute(self.ndarray[r * n_part : r * n_part + r, ...], n_head, n_head_kv)) |
| 474 | |
| 475 | def part(self, n_part: int) -> UnquantizedTensor: |
| 476 | r = self.ndarray.shape[0] // 3 |
| 477 | return UnquantizedTensor(self.ndarray[r * n_part : r * n_part + r, ...]) |
| 478 | |
| 479 | def permute(self, n_head: int, n_head_kv: int) -> UnquantizedTensor: |
| 480 | return UnquantizedTensor(permute(self.ndarray, n_head, n_head_kv)) |
| 481 | |
| 482 | |
| 483 | def load_unquantized(lazy_tensor: LazyTensor, expected_dtype: Any = None, convert: bool = False) -> NDArray: |