MCPcopy Create free account
hub / github.com/LMCache/LMCache / _normalize_lmcache_objects

Function _normalize_lmcache_objects

lmcache/python_ops_fallback.py:1034–1085  ·  view source on GitHub ↗

Normalize LMCache object inputs to chunk tensors. Accepts either a list of chunk tensors or a ``list[int]`` of raw CPU pointers. When a pointer list is provided *shape_desc*, *lmcache_chunk_size*, *engine_kv_format*, and *dtype* must be supplied so the tensors can be reconstructed v

(
    lmcache_objects_ptrs: "list[int] | list[torch.Tensor]",
    shape_desc: "PageBufferShapeDesc | None" = None,
    lmcache_chunk_size: "int | None" = None,
    engine_kv_format: "EngineKVFormat | None" = None,
    dtype: "torch.dtype | None" = None,
)

Source from the content-addressed store, hash-verified

1032
1033
1034def _normalize_lmcache_objects(
1035 lmcache_objects_ptrs: "list[int] | list[torch.Tensor]",
1036 shape_desc: "PageBufferShapeDesc | None" = None,
1037 lmcache_chunk_size: "int | None" = None,
1038 engine_kv_format: "EngineKVFormat | None" = None,
1039 dtype: "torch.dtype | None" = None,
1040) -> list[torch.Tensor]:
1041 """Normalize LMCache object inputs to chunk tensors.
1042
1043 Accepts either a list of chunk tensors or a ``list[int]`` of raw CPU pointers.
1044 When a pointer list is provided *shape_desc*, *lmcache_chunk_size*,
1045 *engine_kv_format*, and *dtype* must be supplied so the tensors can be
1046 reconstructed via :func:`_tensor_from_ptr` on the CPU.
1047 """
1048 if not isinstance(lmcache_objects_ptrs, list):
1049 raise TypeError(
1050 "lmcache_objects_ptrs must be a list[torch.Tensor] or list[int]; "
1051 "got: " + type(lmcache_objects_ptrs).__name__
1052 )
1053 if not lmcache_objects_ptrs:
1054 return []
1055 if isinstance(lmcache_objects_ptrs[0], torch.Tensor):
1056 return lmcache_objects_ptrs # type: ignore[return-value]
1057 if isinstance(lmcache_objects_ptrs[0], int):
1058 # Pointer mode: reconstruct chunk tensors (always on CPU).
1059 if (
1060 shape_desc is None
1061 or lmcache_chunk_size is None
1062 or engine_kv_format is None
1063 or dtype is None
1064 ):
1065 raise ValueError(
1066 "_normalize_lmcache_objects: shape_desc, lmcache_chunk_size, "
1067 "engine_kv_format, and dtype are required when lmcache_objects_ptrs "
1068 "contains raw int pointers"
1069 )
1070 nl = int(shape_desc.nl)
1071 nh = int(shape_desc.nh)
1072 hs = int(shape_desc.hs)
1073 chunk_tokens = lmcache_chunk_size
1074 if _is_mla_format(engine_kv_format):
1075 chunk_shape: tuple[int, ...] = (nl, chunk_tokens, hs)
1076 else:
1077 chunk_shape = (2, nl, chunk_tokens, nh * hs)
1078 return [
1079 _tensor_from_ptr(ptr, chunk_shape, dtype, "cpu")
1080 for ptr in lmcache_objects_ptrs
1081 ]
1082 raise TypeError(
1083 "lmcache_objects_ptrs must be a list[torch.Tensor] or list[int]; "
1084 "got list containing: " + type(lmcache_objects_ptrs[0]).__name__
1085 )
1086
1087
1088def _to_block_id_list(block_ids: torch.Tensor | list[int]) -> list[int]:

Callers 1

Calls 2

_is_mla_formatFunction · 0.85
_tensor_from_ptrFunction · 0.85

Tested by

no test coverage detected