(self, config, tokenizer_config, quant, n_layers, use_mla, bsize)
| 45 | |
| 46 | class Metadata: |
| 47 | def __init__(self, config, tokenizer_config, quant, n_layers, use_mla, bsize): |
| 48 | arch = config["architectures"][0] |
| 49 | if arch not in SUPPORTED_ARCHITECTURES: |
| 50 | raise Exception(f"Architecture {arch} is not supported, must be one of {SUPPORTED_ARCHITECTURES}") |
| 51 | self.arch = arch |
| 52 | self.use_mla = bool(use_mla) |
| 53 | if quant not in SUPPORTED_QUANTS: |
| 54 | raise Exception(f"Quantization {quant} is not supported, must be one of {SUPPORTED_QUANTS}") |
| 55 | self.quant: Quant = SUPPORTED_QUANTS[quant] |
| 56 | if isinstance(self.quant, BlockQuant): |
| 57 | is_bsize_configurable = self.quant.block_size is not None |
| 58 | if is_bsize_configurable and bsize is not None: |
| 59 | self.quant.block_size = (bsize, bsize) |
| 60 | if arch in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: |
| 61 | self.dim = config["hidden_size"] |
| 62 | self.hidden_dim = config["intermediate_size"] |
| 63 | self.n_layers = config["num_hidden_layers"] |
| 64 | if n_layers is not None and self.n_layers > n_layers: |
| 65 | self.n_layers = n_layers |
| 66 | self.n_heads = config["num_attention_heads"] |
| 67 | self.vocab_size = config["vocab_size"] |
| 68 | self.max_seq_len = tokenizer_config["model_max_length"] |
| 69 | self.bos_token_id = config["bos_token_id"] |
| 70 | self.eos_token_id = config["eos_token_id"] |
| 71 | self.rope_theta = config.get("rope_theta", 10000.0) |
| 72 | self.norm_eps = config["rms_norm_eps"] |
| 73 | self.norm_type = "rmsnorm" |
| 74 | |
| 75 | # quantization |
| 76 | self.original_quantization_config = config.get("quantization_config", None) |
| 77 | if self.original_quantization_config is not None: |
| 78 | dequant_block_sizes = self.original_quantization_config["weight_block_size"] |
| 79 | assert type(dequant_block_sizes) == list and len(dequant_block_sizes) == 2 |
| 80 | assert self.original_quantization_config["quant_method"] == "fp8" |
| 81 | |
| 82 | assert config.get("attention_bias", False) == False |
| 83 | assert config.get("mlp_bias", False) == False |
| 84 | |
| 85 | assert config["hidden_act"] in ["gelu", "silu"] |
| 86 | self.act_type = config["hidden_act"] |
| 87 | self.first_k_dense_replace = config["first_k_dense_replace"] |
| 88 | |
| 89 | # multi-latent attention |
| 90 | self.kv_lora_rank = config["kv_lora_rank"] |
| 91 | self.q_lora_rank = config["q_lora_rank"] or 0 |
| 92 | if self.use_mla: |
| 93 | # TODO: support MLA with q_lora_rank == 0 (DeepSeek V2 Lite) |
| 94 | assert self.q_lora_rank > 0 and self.kv_lora_rank > 0 |
| 95 | self.qk_nope_head_dim = config["qk_nope_head_dim"] |
| 96 | self.qk_rope_head_dim = config["qk_rope_head_dim"] |
| 97 | self.v_head_dim = config["v_head_dim"] |
| 98 | |
| 99 | # mixture of experts |
| 100 | self.n_shared_experts = config["n_shared_experts"] |
| 101 | self.n_routed_experts = config["n_routed_experts"] |
| 102 | self.n_active_routed = config["num_experts_per_tok"] |
| 103 | self.moe_intermediate_size = config["moe_intermediate_size"] |
| 104 | self.routed_scaling_factor = config["routed_scaling_factor"] |
nothing calls this directly
no outgoing calls
no test coverage detected