Normalize LMCache object inputs to chunk tensors. Accepts either a list of chunk tensors or a ``list[int]`` of raw CPU pointers. When a pointer list is provided *shape_desc*, *lmcache_chunk_size*, *engine_kv_format*, and *dtype* must be supplied so the tensors can be reconstructed v
(
lmcache_objects_ptrs: "list[int] | list[torch.Tensor]",
shape_desc: "PageBufferShapeDesc | None" = None,
lmcache_chunk_size: "int | None" = None,
engine_kv_format: "EngineKVFormat | None" = None,
dtype: "torch.dtype | None" = None,
)
| 1032 | |
| 1033 | |
| 1034 | def _normalize_lmcache_objects( |
| 1035 | lmcache_objects_ptrs: "list[int] | list[torch.Tensor]", |
| 1036 | shape_desc: "PageBufferShapeDesc | None" = None, |
| 1037 | lmcache_chunk_size: "int | None" = None, |
| 1038 | engine_kv_format: "EngineKVFormat | None" = None, |
| 1039 | dtype: "torch.dtype | None" = None, |
| 1040 | ) -> list[torch.Tensor]: |
| 1041 | """Normalize LMCache object inputs to chunk tensors. |
| 1042 | |
| 1043 | Accepts either a list of chunk tensors or a ``list[int]`` of raw CPU pointers. |
| 1044 | When a pointer list is provided *shape_desc*, *lmcache_chunk_size*, |
| 1045 | *engine_kv_format*, and *dtype* must be supplied so the tensors can be |
| 1046 | reconstructed via :func:`_tensor_from_ptr` on the CPU. |
| 1047 | """ |
| 1048 | if not isinstance(lmcache_objects_ptrs, list): |
| 1049 | raise TypeError( |
| 1050 | "lmcache_objects_ptrs must be a list[torch.Tensor] or list[int]; " |
| 1051 | "got: " + type(lmcache_objects_ptrs).__name__ |
| 1052 | ) |
| 1053 | if not lmcache_objects_ptrs: |
| 1054 | return [] |
| 1055 | if isinstance(lmcache_objects_ptrs[0], torch.Tensor): |
| 1056 | return lmcache_objects_ptrs # type: ignore[return-value] |
| 1057 | if isinstance(lmcache_objects_ptrs[0], int): |
| 1058 | # Pointer mode: reconstruct chunk tensors (always on CPU). |
| 1059 | if ( |
| 1060 | shape_desc is None |
| 1061 | or lmcache_chunk_size is None |
| 1062 | or engine_kv_format is None |
| 1063 | or dtype is None |
| 1064 | ): |
| 1065 | raise ValueError( |
| 1066 | "_normalize_lmcache_objects: shape_desc, lmcache_chunk_size, " |
| 1067 | "engine_kv_format, and dtype are required when lmcache_objects_ptrs " |
| 1068 | "contains raw int pointers" |
| 1069 | ) |
| 1070 | nl = int(shape_desc.nl) |
| 1071 | nh = int(shape_desc.nh) |
| 1072 | hs = int(shape_desc.hs) |
| 1073 | chunk_tokens = lmcache_chunk_size |
| 1074 | if _is_mla_format(engine_kv_format): |
| 1075 | chunk_shape: tuple[int, ...] = (nl, chunk_tokens, hs) |
| 1076 | else: |
| 1077 | chunk_shape = (2, nl, chunk_tokens, nh * hs) |
| 1078 | return [ |
| 1079 | _tensor_from_ptr(ptr, chunk_shape, dtype, "cpu") |
| 1080 | for ptr in lmcache_objects_ptrs |
| 1081 | ] |
| 1082 | raise TypeError( |
| 1083 | "lmcache_objects_ptrs must be a list[torch.Tensor] or list[int]; " |
| 1084 | "got list containing: " + type(lmcache_objects_ptrs[0]).__name__ |
| 1085 | ) |
| 1086 | |
| 1087 | |
| 1088 | def _to_block_id_list(block_ids: torch.Tensor | list[int]) -> list[int]: |
no test coverage detected