Copy model parameters into a (differently-shaped) list of full-precision parameters.
(param_groups_and_shapes)
| 33 | |
| 34 | |
| 35 | def make_master_params(param_groups_and_shapes): |
| 36 | """ |
| 37 | Copy model parameters into a (differently-shaped) list of full-precision |
| 38 | parameters. |
| 39 | """ |
| 40 | master_params = [] |
| 41 | for param_group, shape in param_groups_and_shapes: |
| 42 | master_param = nn.Parameter( |
| 43 | _flatten_dense_tensors( |
| 44 | [param.detach().float() for (_, param) in param_group] |
| 45 | ).view(shape) |
| 46 | ) |
| 47 | master_param.requires_grad = True |
| 48 | master_params.append(master_param) |
| 49 | return master_params |
| 50 | |
| 51 | |
| 52 | def model_grads_to_master_grads(param_groups_and_shapes, master_params): |
no outgoing calls
no test coverage detected