| 157 | |
| 158 | |
| 159 | def get_sequential_groups(model): |
| 160 | if model.config.model_type in LLAMA_LIKE: |
| 161 | assert "mixtral" not in model.config.model_type.lower() # check that this is not mixtral |
| 162 | return [ |
| 163 | ["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj"], |
| 164 | ["self_attn.o_proj"], |
| 165 | ["mlp.up_proj", "mlp.gate_proj"], |
| 166 | ["mlp.down_proj"], |
| 167 | ] |
| 168 | elif model.config.model_type.lower() in FALCON_TYPES: |
| 169 | return [ |
| 170 | ["self_attention.query_key_value"], |
| 171 | ["self_attention.dense"], |
| 172 | ["mlp.dense_h_to_4h"], |
| 173 | ["mlp.dense_4h_to_h"], |
| 174 | ] |
| 175 | elif model.config.model_type == "opt": |
| 176 | return [ |
| 177 | ["self_attn.q_proj"], |
| 178 | ["self_attn.k_proj"], |
| 179 | ["self_attn.v_proj"], |
| 180 | ["self_attn.out_proj"], |
| 181 | ["fc1"], |
| 182 | ["fc2"], |
| 183 | ] |
| 184 | elif model.config.model_type == "phi3": |
| 185 | return [["self_attn.qkv_proj"], ["self_attn.o_proj"], ["mlp.gate_up_proj"], ["mlp.down_proj"]] |
| 186 | else: |
| 187 | raise ValueError(MODEL_ERROR_MSG.format(model.config.model_type)) |
| 188 | |
| 189 | |
| 190 | def read_quant_weight_from_file(load_path, block_i, layer_name, device): |