Initialize distributed training environment.
(args)
| 243 | |
| 244 | |
| 245 | def distributed_init(args): |
| 246 | """Initialize distributed training environment.""" |
| 247 | world_size = int(os.getenv('WORLD_SIZE', '1')) |
| 248 | rank = int(os.getenv('RANK', '0')) |
| 249 | local_rank = args.local_rank |
| 250 | |
| 251 | # Set environment variables for distributed training |
| 252 | os.environ['MASTER_ADDR'] = os.getenv('MASTER_ADDR', 'localhost') |
| 253 | os.environ['MASTER_PORT'] = str(args.master_port) |
| 254 | |
| 255 | return world_size, rank, local_rank |
| 256 | |
| 257 | |
| 258 | def get_prodigy_d(optimizer): |