(self)
| 2201 | return RandomLTDScheduler(configs) |
| 2202 | |
| 2203 | def _configure_quantization(self): |
| 2204 | ( |
| 2205 | quantize_weight_in_forward, |
| 2206 | quantize_enabled, |
| 2207 | q_groups, |
| 2208 | q_mixed_fp16, |
| 2209 | q_change_ratio, |
| 2210 | q_type, |
| 2211 | q_rounding, |
| 2212 | q_verbose, |
| 2213 | use_quantizer_kernel, |
| 2214 | ) = self.quantize_training() |
| 2215 | if quantize_enabled and not quantize_weight_in_forward: |
| 2216 | assert self.fp16_enabled( |
| 2217 | ), "MoQ (quantize in optimization step) weight quantization is only supported for FP16" |
| 2218 | quantizer = None |
| 2219 | if quantize_enabled and not quantize_weight_in_forward: |
| 2220 | from deepspeed.runtime.quantize import Quantizer |
| 2221 | |
| 2222 | quantizer = Quantizer( |
| 2223 | q_groups, |
| 2224 | q_mixed_fp16, |
| 2225 | q_change_ratio, |
| 2226 | q_type, |
| 2227 | q_rounding, |
| 2228 | q_verbose, |
| 2229 | self.eigenvalue_enabled(), |
| 2230 | use_quantizer_kernel, |
| 2231 | self.eigenvalue_layer_num() if self.eigenvalue_enabled() else 0, |
| 2232 | ) |
| 2233 | return quantizer |
| 2234 | |
| 2235 | def _configure_fp16_optimizer(self, optimizer, low_precision_dtype): |
| 2236 | dynamic_loss_args = self.dynamic_loss_scale_args() |
no test coverage detected