Build a small LLaMA 2 model from scratch, then convert to BitNet.
(tokenizer, cfg: dict)
| 153 | # ────────────────────────────────────────────────────────────────────────────── |
| 154 | |
| 155 | def build_model(tokenizer, cfg: dict) -> LlamaForCausalLM: |
| 156 | """Build a small LLaMA 2 model from scratch, then convert to BitNet.""" |
| 157 | config = LlamaConfig( |
| 158 | vocab_size=tokenizer.vocab_size, |
| 159 | hidden_size=cfg["hidden_size"], |
| 160 | intermediate_size=cfg["intermediate_size"], |
| 161 | num_hidden_layers=cfg["num_hidden_layers"], |
| 162 | num_attention_heads=cfg["num_attention_heads"], |
| 163 | max_position_embeddings=cfg["max_position_embeddings"], |
| 164 | rms_norm_eps=1e-6, |
| 165 | # Paper: no bias terms |
| 166 | attention_bias=False, |
| 167 | mlp_bias=False, |
| 168 | ) |
| 169 | model = LlamaForCausalLM(config) |
| 170 | model = convert_to_bitnet(model) |
| 171 | return model |
| 172 | |
| 173 | |
| 174 | # ────────────────────────────────────────────────────────────────────────────── |
no test coverage detected