| 38 | |
| 39 | @dataclass |
| 40 | class ModelArgs: |
| 41 | dim: int = 2560 |
| 42 | n_layers: int = 30 |
| 43 | n_heads: int = 20 |
| 44 | n_kv_heads: int = 5 |
| 45 | vocab_size: int = 128256 |
| 46 | ffn_dim: int = 6912 |
| 47 | norm_eps: float = 1e-5 |
| 48 | rope_theta: float = 500000.0 |
| 49 | use_kernel: bool = False |
| 50 | |
| 51 | |
| 52 | LayerCache = Tuple[torch.Tensor, torch.Tensor] |
nothing calls this directly
no outgoing calls
no test coverage detected