| 596 | |
| 597 | |
| 598 | class UnquantizedTensor(Tensor): |
| 599 | def __init__(self, ndarray: NDArray) -> None: |
| 600 | assert isinstance(ndarray, np.ndarray) |
| 601 | self.ndarray = ndarray |
| 602 | self.data_type = NUMPY_TYPE_TO_DATA_TYPE[ndarray.dtype] |
| 603 | |
| 604 | def astype(self, data_type: DataType) -> Tensor: |
| 605 | dtype = data_type.dtype |
| 606 | if self.data_type == DT_BF16: |
| 607 | self.ndarray = bf16_to_fp32(self.ndarray) |
| 608 | return UnquantizedTensor(self.ndarray.astype(dtype)) |
| 609 | |
| 610 | def to_ggml(self) -> UnquantizedTensor: |
| 611 | return self |
| 612 | |
| 613 | def permute_part(self, n_part: int, n_head: int, n_head_kv: int) -> UnquantizedTensor: |
| 614 | r = self.ndarray.shape[0] // 3 |
| 615 | return UnquantizedTensor(permute(self.ndarray[r * n_part : r * n_part + r, ...], n_head, n_head_kv)) |
| 616 | |
| 617 | def part(self, n_part: int) -> UnquantizedTensor: |
| 618 | r = self.ndarray.shape[0] // 3 |
| 619 | return UnquantizedTensor(self.ndarray[r * n_part : r * n_part + r, ...]) |
| 620 | |
| 621 | def permute(self, n_head: int, n_head_kv: int) -> UnquantizedTensor: |
| 622 | return UnquantizedTensor(permute(self.ndarray, n_head, n_head_kv)) |
| 623 | |
| 624 | |
| 625 | def load_unquantized(lazy_tensor: LazyTensor, expected_dtype: Any = None, convert: bool = False) -> NDArray: |