| 146 | logger.handlers[0].flush() |
| 147 | |
| 148 | def debug_param(name, param): |
| 149 | print_all(f"param: {name}, min: {param.min()}, max: {param.max()}, mean: {param.mean()}, std: {param.std()}, scale: {param.abs().mean()}, first5: {param.flatten()[:5]}, last5: {param.flatten()[-5:]}") |
| 150 | from deepspeed.utils import safe_get_full_grad, safe_get_full_optimizer_state |
| 151 | g = safe_get_full_grad(param) |
| 152 | if g is not None: |
| 153 | print_all(f"grad: {name}, min: {g.min()}, max: {g.max()}, mean: {g.mean()}, std: {g.std()}, scale: {g.abs().mean()}, first5: {g.flatten()[:5]}, last5: {param.flatten()[-5:]}") |
| 154 | s = safe_get_full_optimizer_state(param, 'exp_avg') |
| 155 | if s is not None: |
| 156 | print_all(f"state: {name}, min: {s.min()}, max: {s.max()}, mean: {s.mean()}, std: {s.std()}, scale: {s.abs().mean()}, first5: {s.flatten()[:5]}") |
| 157 | |
| 158 | def get_free_port(): |
| 159 | import socket |