| 34 | |
| 35 | @dataclass |
| 36 | class GqlaLayout: |
| 37 | num_heads: int # H |
| 38 | num_kv_heads: int # G |
| 39 | group_size: int # gs = H / G |
| 40 | qk_nope: int # d_k |
| 41 | qk_rope: int |
| 42 | v_dim: int # d_v |
| 43 | kv_lora: int |
| 44 | |
| 45 | @classmethod |
| 46 | def from_config(cls, config, num_kv_heads: int) -> "GqlaLayout": |
| 47 | H = config.num_attention_heads |
| 48 | if H % num_kv_heads != 0: |
| 49 | raise ValueError( |
| 50 | f"num_attention_heads ({H}) not divisible by num_kv_heads ({num_kv_heads})" |
| 51 | ) |
| 52 | return cls( |
| 53 | num_heads=H, |
| 54 | num_kv_heads=num_kv_heads, |
| 55 | group_size=H // num_kv_heads, |
| 56 | qk_nope=config.qk_nope_head_dim, |
| 57 | qk_rope=config.qk_rope_head_dim, |
| 58 | v_dim=config.v_head_dim, |
| 59 | kv_lora=config.kv_lora_rank, |
| 60 | ) |
| 61 | |
| 62 | |
| 63 | # ----------------------------- calibration data ----------------------------- |
nothing calls this directly
no outgoing calls
no test coverage detected