(self)
| 2120 | return RandomLTDScheduler(configs) |
| 2121 | |
| 2122 | def _configure_quantization(self): |
| 2123 | ( |
| 2124 | quantize_weight_in_forward, |
| 2125 | quantize_enabled, |
| 2126 | q_groups, |
| 2127 | q_mixed_fp16, |
| 2128 | q_change_ratio, |
| 2129 | q_type, |
| 2130 | q_rounding, |
| 2131 | q_verbose, |
| 2132 | use_quantizer_kernel, |
| 2133 | ) = self.quantize_training() |
| 2134 | if quantize_enabled and not quantize_weight_in_forward: |
| 2135 | assert self.fp16_enabled( |
| 2136 | ), "MoQ (quantize in optimization step) weight quantization is only supported for FP16" |
| 2137 | quantizer = None |
| 2138 | if quantize_enabled and not quantize_weight_in_forward: |
| 2139 | from deepspeed.runtime.quantize import Quantizer |
| 2140 | |
| 2141 | quantizer = Quantizer( |
| 2142 | q_groups, |
| 2143 | q_mixed_fp16, |
| 2144 | q_change_ratio, |
| 2145 | q_type, |
| 2146 | q_rounding, |
| 2147 | q_verbose, |
| 2148 | self.eigenvalue_enabled(), |
| 2149 | use_quantizer_kernel, |
| 2150 | self.eigenvalue_layer_num() if self.eigenvalue_enabled() else 0, |
| 2151 | ) |
| 2152 | return quantizer |
| 2153 | |
| 2154 | def _configure_fp16_optimizer(self, optimizer, low_precision_dtype): |
| 2155 | dynamic_loss_args = self.dynamic_loss_scale_args() |
no test coverage detected