| 822 | |
| 823 | # generate dummy model |
| 824 | def generate_tensors(self) -> Iterator[tuple[str, np.ndarray]]: |
| 825 | hp_config = model_config[self.params] |
| 826 | hidden_size = hp_config["hidden_size"] |
| 827 | intermediate_size = hp_config["intermediate_size"] |
| 828 | num_hidden_layers = hp_config["num_hidden_layers"] |
| 829 | num_attention_heads = hp_config["num_attention_heads"] |
| 830 | |
| 831 | # generate dummy tensors |
| 832 | tensor = torch.randn((32002, hidden_size), dtype=torch.float32) |
| 833 | yield ("model.embed_tokens.weight", tensor) |
| 834 | for i in range(num_hidden_layers): |
| 835 | yield f"model.layers.{i}.input_layernorm.weight", torch.randn((hidden_size,), dtype=torch.float32) |
| 836 | yield f"model.layers.{i}.mlp.down_proj.weight", torch.randn((hidden_size, intermediate_size), dtype=torch.float32) |
| 837 | yield f"model.layers.{i}.mlp.ffn_layernorm.weight", torch.randn((intermediate_size,), dtype=torch.float32) |
| 838 | yield f"model.layers.{i}.mlp.gate_proj.weight", torch.randn((intermediate_size, hidden_size), dtype=torch.float32) |
| 839 | yield f"model.layers.{i}.mlp.up_proj.weight", torch.randn((intermediate_size, hidden_size), dtype=torch.float32) |
| 840 | yield f"model.layers.{i}.post_attention_layernorm.weight", torch.randn((hidden_size), dtype=torch.float32) |
| 841 | yield f"model.layers.{i}.self_attn.inner_attn_ln.weight", torch.randn((hidden_size,), dtype=torch.float32) |
| 842 | yield f"model.layers.{i}.self_attn.k_proj.weight", torch.randn((hidden_size, hidden_size), dtype=torch.float32) |
| 843 | yield f"model.layers.{i}.self_attn.o_proj.weight", torch.randn((hidden_size, hidden_size), dtype=torch.float32) |
| 844 | yield f"model.layers.{i}.self_attn.q_proj.weight", torch.randn((hidden_size, hidden_size), dtype=torch.float32) |
| 845 | yield f"model.layers.{i}.self_attn.rotary_emb.inv_freq", torch.randn((hidden_size // (num_attention_heads * 2),), dtype=torch.float32) |
| 846 | yield f"model.layers.{i}.self_attn.v_proj.weight", torch.randn((hidden_size, hidden_size), dtype=torch.float32) |
| 847 | tensor = torch.randn((hidden_size,), dtype=torch.float32) |
| 848 | yield("model.norm.weight", tensor) |
| 849 | |
| 850 | |
| 851 | |