Handle cross-layer formats: single tensor [NB, NL, 2, ...].
(
paged_tensor: torch.Tensor,
object_tensors: list[torch.Tensor],
block_ids: torch.Tensor | list[int],
n_block_ids: int,
blocks_per_object: int,
block_size: int,
engine_kv_format: EngineKVFormat,
is_d2h: bool,
skip_prefix_n_blocks: int,
)
| 1278 | |
| 1279 | |
| 1280 | def _transfer_cross_layer( |
| 1281 | paged_tensor: torch.Tensor, |
| 1282 | object_tensors: list[torch.Tensor], |
| 1283 | block_ids: torch.Tensor | list[int], |
| 1284 | n_block_ids: int, |
| 1285 | blocks_per_object: int, |
| 1286 | block_size: int, |
| 1287 | engine_kv_format: EngineKVFormat, |
| 1288 | is_d2h: bool, |
| 1289 | skip_prefix_n_blocks: int, |
| 1290 | ) -> None: |
| 1291 | """Handle cross-layer formats: single tensor [NB, NL, 2, ...].""" |
| 1292 | # NHD: [NB, NL, 2, BS, NH, HS] HND: [NB, NL, 2, NH, BS, HS] |
| 1293 | is_hnd = int(engine_kv_format) == int(EngineKVFormat.NB_NL_TWO_NH_BS_HS) |
| 1294 | num_layers = paged_tensor.shape[1] |
| 1295 | |
| 1296 | if is_hnd: |
| 1297 | # [NB, NL, 2, NH, BS, HS] |
| 1298 | nh = paged_tensor.shape[3] |
| 1299 | hs = paged_tensor.shape[5] |
| 1300 | else: |
| 1301 | # [NB, NL, 2, BS, NH, HS] |
| 1302 | nh = paged_tensor.shape[4] |
| 1303 | hs = paged_tensor.shape[5] |
| 1304 | |
| 1305 | # H2D: pre-transfer objects to paged device |
| 1306 | if not is_d2h and object_tensors: |
| 1307 | objs_on_device = [obj.to(paged_tensor.device) for obj in object_tensors] |
| 1308 | block_ids_dev = torch.as_tensor( |
| 1309 | block_ids, dtype=torch.long, device=paged_tensor.device |
| 1310 | ) |
| 1311 | |
| 1312 | for object_idx, obj in enumerate(object_tensors): |
| 1313 | valid = _valid_block_range_indices( |
| 1314 | object_idx, |
| 1315 | n_block_ids, |
| 1316 | blocks_per_object, |
| 1317 | block_size, |
| 1318 | skip_prefix_n_blocks, |
| 1319 | ) |
| 1320 | if valid is None: |
| 1321 | continue |
| 1322 | idx_start, idx_end, offset_in_object = valid |
| 1323 | n_valid = idx_end - idx_start |
| 1324 | token_end = offset_in_object + n_valid * block_size |
| 1325 | eff_idx = block_ids_dev[idx_start:idx_end] |
| 1326 | |
| 1327 | if is_d2h: |
| 1328 | selected = paged_tensor.index_select(0, eff_idx) |
| 1329 | |
| 1330 | for layer_idx in range(num_layers): |
| 1331 | for kv in range(2): |
| 1332 | if is_d2h: |
| 1333 | slice_t = selected[:, layer_idx, kv] |
| 1334 | if is_hnd: |
| 1335 | # N=n_valid, BS=block_size: |
| 1336 | # [N, NH, BS, HS] -> [N, BS, NH, HS] -> [N*BS, NH*HS] |
| 1337 | flat = slice_t.permute(0, 2, 1, 3).reshape( |
no test coverage detected