()
| 10 | |
| 11 | |
| 12 | def get_4bit_config(): |
| 13 | return transformers.BitsAndBytesConfig( |
| 14 | load_in_4bit=True, |
| 15 | load_in_8bit=False, |
| 16 | llm_int8_threshold=6.0, |
| 17 | llm_int8_has_fp16_weight=False, |
| 18 | bnb_4bit_compute_dtype=torch.float16, |
| 19 | bnb_4bit_use_double_quant=True, |
| 20 | bnb_4bit_quant_type="nf4", |
| 21 | ) |
| 22 | |
| 23 | |
| 24 | def get_model_and_tokenizer(config): |
no outgoing calls
no test coverage detected