(self)
| 2193 | return RandomLTDScheduler(configs) |
| 2194 | |
| 2195 | def _configure_quantization(self): |
| 2196 | ( |
| 2197 | quantize_weight_in_forward, |
| 2198 | quantize_enabled, |
| 2199 | q_groups, |
| 2200 | q_mixed_fp16, |
| 2201 | q_change_ratio, |
| 2202 | q_type, |
| 2203 | q_rounding, |
| 2204 | q_verbose, |
| 2205 | use_quantizer_kernel, |
| 2206 | ) = self.quantize_training() |
| 2207 | if quantize_enabled and not quantize_weight_in_forward: |
| 2208 | assert self.fp16_enabled( |
| 2209 | ), "MoQ (quantize in optimization step) weight quantization is only supported for FP16" |
| 2210 | quantizer = None |
| 2211 | if quantize_enabled and not quantize_weight_in_forward: |
| 2212 | from deepspeed.runtime.quantize import Quantizer |
| 2213 | |
| 2214 | quantizer = Quantizer( |
| 2215 | q_groups, |
| 2216 | q_mixed_fp16, |
| 2217 | q_change_ratio, |
| 2218 | q_type, |
| 2219 | q_rounding, |
| 2220 | q_verbose, |
| 2221 | self.eigenvalue_enabled(), |
| 2222 | use_quantizer_kernel, |
| 2223 | self.eigenvalue_layer_num() if self.eigenvalue_enabled() else 0, |
| 2224 | ) |
| 2225 | return quantizer |
| 2226 | |
| 2227 | def _configure_fp16_optimizer(self, optimizer, low_precision_dtype): |
| 2228 | dynamic_loss_args = self.dynamic_loss_scale_args() |
no test coverage detected