| 17311 | * persistent caches used by decode. Keeping this separate from decode |
| 17312 | * avoids a slow loop of one-token graph steps for long prompts. */ |
| 17313 | /* Class E — embedding-tier-only prompt-token integer buffer. |
| 17314 | * Captured at metal_graph_alloc_raw_cap time from placement[0] (or 0 in |
| 17315 | * single-tier / diagnostic paths). Non-embedding slots stay NULL. Readers |
| 17316 | * go through metal_graph_prefill_tokens() below. */ |
| 17317 | ds4_gpu_tensor *prefill_tokens_by_tier[DS4_MAX_GPUS]; |
| 17318 | int emb_tier; |
| 17319 | /* Class P batch (chunked-prefill) scratch — per-tier |
| 17320 | * replicated. The cur/next pair is ping-ponged per layer step on the |
| 17321 | * layer's active tier; tier transitions copy the active buffer across |
| 17322 | * boundaries via ds4_gpu_tensor_copy_xdev (handled in B6). */ |
| 17323 | ds4_gpu_tensor *batch_cur_hc_by_tier[DS4_MAX_GPUS]; |
| 17324 | ds4_gpu_tensor *batch_next_hc_by_tier[DS4_MAX_GPUS]; |
| 17325 | ds4_gpu_tensor *batch_flat_hc_by_tier[DS4_MAX_GPUS]; |
| 17326 | ds4_gpu_tensor *batch_hc_mix_by_tier[DS4_MAX_GPUS]; |
| 17327 | ds4_gpu_tensor *batch_hc_split_by_tier[DS4_MAX_GPUS]; |
| 17328 | ds4_gpu_tensor *batch_attn_cur_by_tier[DS4_MAX_GPUS]; |
| 17329 | ds4_gpu_tensor *batch_attn_norm_by_tier[DS4_MAX_GPUS]; |
| 17330 | ds4_gpu_tensor *batch_qr_by_tier[DS4_MAX_GPUS]; |
| 17331 | ds4_gpu_tensor *batch_qr_norm_by_tier[DS4_MAX_GPUS]; |
| 17332 | ds4_gpu_tensor *batch_q_by_tier[DS4_MAX_GPUS]; |
| 17333 | ds4_gpu_tensor *batch_kv_raw_by_tier[DS4_MAX_GPUS]; |
| 17334 | ds4_gpu_tensor *batch_kv_by_tier[DS4_MAX_GPUS]; |
| 17335 | ds4_gpu_tensor *batch_comp_kv_by_tier[DS4_MAX_GPUS]; |
| 17336 | ds4_gpu_tensor *batch_comp_sc_by_tier[DS4_MAX_GPUS]; |
| 17337 | ds4_gpu_tensor *batch_indexer_q_by_tier[DS4_MAX_GPUS]; |
| 17338 | ds4_gpu_tensor *batch_indexer_weights_by_tier[DS4_MAX_GPUS]; |
| 17339 | ds4_gpu_tensor *batch_heads_by_tier[DS4_MAX_GPUS]; |
| 17340 | ds4_gpu_tensor *batch_attn_low_by_tier[DS4_MAX_GPUS]; |
| 17341 | ds4_gpu_tensor *batch_attn_out_by_tier[DS4_MAX_GPUS]; |
| 17342 | ds4_gpu_tensor *batch_group_tmp_by_tier[DS4_MAX_GPUS]; |
| 17343 | ds4_gpu_tensor *batch_low_tmp_by_tier[DS4_MAX_GPUS]; |
| 17344 | ds4_gpu_tensor *batch_after_attn_hc_by_tier[DS4_MAX_GPUS]; |
| 17345 | ds4_gpu_tensor *batch_ffn_cur_by_tier[DS4_MAX_GPUS]; |
| 17346 | ds4_gpu_tensor *batch_ffn_norm_by_tier[DS4_MAX_GPUS]; |
| 17347 | ds4_gpu_tensor *batch_shared_gate_by_tier[DS4_MAX_GPUS]; |
| 17348 | ds4_gpu_tensor *batch_shared_up_by_tier[DS4_MAX_GPUS]; |
| 17349 | ds4_gpu_tensor *batch_shared_mid_by_tier[DS4_MAX_GPUS]; |
| 17350 | ds4_gpu_tensor *batch_shared_out_by_tier[DS4_MAX_GPUS]; |
| 17351 | ds4_gpu_tensor *batch_router_logits_by_tier[DS4_MAX_GPUS]; |
| 17352 | ds4_gpu_tensor *batch_router_probs_by_tier[DS4_MAX_GPUS]; |
| 17353 | ds4_gpu_tensor *batch_router_selected_by_tier[DS4_MAX_GPUS]; |
| 17354 | ds4_gpu_tensor *batch_router_weights_by_tier[DS4_MAX_GPUS]; |
| 17355 | ds4_gpu_tensor *batch_routed_gate_by_tier[DS4_MAX_GPUS]; |
| 17356 | ds4_gpu_tensor *batch_routed_up_by_tier[DS4_MAX_GPUS]; |
| 17357 | ds4_gpu_tensor *batch_routed_mid_by_tier[DS4_MAX_GPUS]; |
| 17358 | ds4_gpu_tensor *batch_routed_down_by_tier[DS4_MAX_GPUS]; |
| 17359 | ds4_gpu_tensor *batch_routed_out_by_tier[DS4_MAX_GPUS]; |
| 17360 | bool batch_routed_mid_is_f16; |
| 17361 | ds4_gpu_tensor *batch_ffn_out_by_tier[DS4_MAX_GPUS]; |
| 17362 | bool owns_prefill_workspace; |
| 17363 | bool materialize_ffn_out; |
| 17364 | /* Class P (replicated per tier — this is |
| 17365 | * consumed in per-layer attn/FFN kernels, NOT embedding-only). Read-only |
| 17366 | * after init; replicate by writing the same host directions buffer to |
| 17367 | * every used tier's slot during session setup. */ |
| 17368 | ds4_gpu_tensor *directional_steering_dirs_by_tier[DS4_MAX_GPUS]; |
| 17369 | float directional_steering_attn_scale; |
| 17370 | float directional_steering_ffn_scale; |
no test coverage detected