(self)
| 2336 | return RandomLTDScheduler(configs) |
| 2337 | |
| 2338 | def _configure_quantization(self): |
| 2339 | ( |
| 2340 | quantize_weight_in_forward, |
| 2341 | quantize_enabled, |
| 2342 | q_groups, |
| 2343 | q_mixed_fp16, |
| 2344 | q_change_ratio, |
| 2345 | q_type, |
| 2346 | q_rounding, |
| 2347 | q_verbose, |
| 2348 | use_quantizer_kernel, |
| 2349 | ) = self.quantize_training() |
| 2350 | if quantize_enabled and not quantize_weight_in_forward: |
| 2351 | assert self.fp16_enabled( |
| 2352 | ), "MoQ (quantize in optimization step) weight quantization is only supported for FP16" |
| 2353 | quantizer = None |
| 2354 | if quantize_enabled and not quantize_weight_in_forward: |
| 2355 | from deepspeed.runtime.quantize import Quantizer |
| 2356 | |
| 2357 | quantizer = Quantizer( |
| 2358 | q_groups, |
| 2359 | q_mixed_fp16, |
| 2360 | q_change_ratio, |
| 2361 | q_type, |
| 2362 | q_rounding, |
| 2363 | q_verbose, |
| 2364 | self.eigenvalue_enabled(), |
| 2365 | use_quantizer_kernel, |
| 2366 | self.eigenvalue_layer_num() if self.eigenvalue_enabled() else 0, |
| 2367 | ) |
| 2368 | return quantizer |
| 2369 | |
| 2370 | def _configure_fp16_optimizer(self, optimizer, low_precision_dtype): |
| 2371 | dynamic_loss_args = self.dynamic_loss_scale_args() |
no test coverage detected