| 158 | |
| 159 | class Float16Module(MegatronModule): |
| 160 | def __init__(self, module, args): |
| 161 | super(Float16Module, self).__init__() |
| 162 | |
| 163 | if args.fp16: |
| 164 | self.add_module("module", module.half()) |
| 165 | |
| 166 | def float16_convertor(val): |
| 167 | return val.half() |
| 168 | |
| 169 | elif args.bf16: |
| 170 | self.add_module("module", module.bfloat16()) |
| 171 | |
| 172 | def float16_convertor(val): |
| 173 | return val.bfloat16() |
| 174 | |
| 175 | else: |
| 176 | raise Exception("should not be here") |
| 177 | |
| 178 | self.float16_convertor = float16_convertor |
| 179 | |
| 180 | def forward(self, *inputs, **kwargs): |
| 181 | if mpu.is_pipeline_first_stage(): |