MCPcopy Create free account
hub / github.com/LMCache/LMCache / _transfer_cross_layer

Function _transfer_cross_layer

lmcache/python_ops_fallback.py:1280–1359  ·  view source on GitHub ↗

Handle cross-layer formats: single tensor [NB, NL, 2, ...].

(
    paged_tensor: torch.Tensor,
    object_tensors: list[torch.Tensor],
    block_ids: torch.Tensor | list[int],
    n_block_ids: int,
    blocks_per_object: int,
    block_size: int,
    engine_kv_format: EngineKVFormat,
    is_d2h: bool,
    skip_prefix_n_blocks: int,
)

Source from the content-addressed store, hash-verified

1278
1279
1280def _transfer_cross_layer(
1281 paged_tensor: torch.Tensor,
1282 object_tensors: list[torch.Tensor],
1283 block_ids: torch.Tensor | list[int],
1284 n_block_ids: int,
1285 blocks_per_object: int,
1286 block_size: int,
1287 engine_kv_format: EngineKVFormat,
1288 is_d2h: bool,
1289 skip_prefix_n_blocks: int,
1290) -> None:
1291 """Handle cross-layer formats: single tensor [NB, NL, 2, ...]."""
1292 # NHD: [NB, NL, 2, BS, NH, HS] HND: [NB, NL, 2, NH, BS, HS]
1293 is_hnd = int(engine_kv_format) == int(EngineKVFormat.NB_NL_TWO_NH_BS_HS)
1294 num_layers = paged_tensor.shape[1]
1295
1296 if is_hnd:
1297 # [NB, NL, 2, NH, BS, HS]
1298 nh = paged_tensor.shape[3]
1299 hs = paged_tensor.shape[5]
1300 else:
1301 # [NB, NL, 2, BS, NH, HS]
1302 nh = paged_tensor.shape[4]
1303 hs = paged_tensor.shape[5]
1304
1305 # H2D: pre-transfer objects to paged device
1306 if not is_d2h and object_tensors:
1307 objs_on_device = [obj.to(paged_tensor.device) for obj in object_tensors]
1308 block_ids_dev = torch.as_tensor(
1309 block_ids, dtype=torch.long, device=paged_tensor.device
1310 )
1311
1312 for object_idx, obj in enumerate(object_tensors):
1313 valid = _valid_block_range_indices(
1314 object_idx,
1315 n_block_ids,
1316 blocks_per_object,
1317 block_size,
1318 skip_prefix_n_blocks,
1319 )
1320 if valid is None:
1321 continue
1322 idx_start, idx_end, offset_in_object = valid
1323 n_valid = idx_end - idx_start
1324 token_end = offset_in_object + n_valid * block_size
1325 eff_idx = block_ids_dev[idx_start:idx_end]
1326
1327 if is_d2h:
1328 selected = paged_tensor.index_select(0, eff_idx)
1329
1330 for layer_idx in range(num_layers):
1331 for kv in range(2):
1332 if is_d2h:
1333 slice_t = selected[:, layer_idx, kv]
1334 if is_hnd:
1335 # N=n_valid, BS=block_size:
1336 # [N, NH, BS, HS] -> [N, BS, NH, HS] -> [N*BS, NH*HS]
1337 flat = slice_t.permute(0, 2, 1, 3).reshape(

Callers 1

Calls 2

reshapeMethod · 0.80

Tested by

no test coverage detected