| 12 | |
| 13 | @dataclass |
| 14 | class ModelArgs: |
| 15 | block_size: int = 4096 |
| 16 | vocab_size: int = 32000 |
| 17 | n_layer: int = 32 |
| 18 | n_head: int = 32 |
| 19 | dim: int = 4096 |
| 20 | intermediate_size: int = None |
| 21 | n_local_heads: int = -1 |
| 22 | head_dim: int = 64 |
| 23 | rope_base: float = 10000 |
| 24 | norm_eps: float = 1e-5 |
| 25 | |
| 26 | def __post_init__(self): |
| 27 | if self.n_local_heads == -1: |
| 28 | self.n_local_heads = self.n_head |
| 29 | if self.intermediate_size is None: |
| 30 | hidden_dim = 4 * self.dim |
| 31 | n_hidden = int(2 * hidden_dim / 3) |
| 32 | self.intermediate_size = n_hidden + (256 - n_hidden % 256) if n_hidden % 256 else n_hidden |
| 33 | self.head_dim = self.dim // self.n_head |
| 34 | |
| 35 | @classmethod |
| 36 | def from_name(cls, name: str): |
| 37 | if name in transformer_configs: |
| 38 | return cls(**transformer_configs[name]) |
| 39 | config = [k for k in transformer_configs if k in name.upper() or k in name] |
| 40 | assert len(config) == 1, f"Unknown model name: {name}" |
| 41 | return cls(**transformer_configs[config[0]]) |
| 42 | |
| 43 | def invert_convert_q(w: torch.Tensor, config: ModelArgs) -> torch.Tensor: |
| 44 | return rearrange(w, '(h l d) i -> (h d l) i', h=config.n_head, l=2) |
nothing calls this directly
no outgoing calls
no test coverage detected