Log detailed memory usage information.
(device, step, phase="unknown")
| 282 | |
| 283 | |
| 284 | def log_memory_usage(device, step, phase="unknown"): |
| 285 | """Log detailed memory usage information.""" |
| 286 | if not torch.cuda.is_available(): |
| 287 | return |
| 288 | |
| 289 | memory_allocated = torch.cuda.memory_allocated(device) / 1e9 |
| 290 | memory_reserved = torch.cuda.memory_reserved(device) / 1e9 |
| 291 | memory_free = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device) |
| 292 | memory_free = memory_free / 1e9 |
| 293 | |
| 294 | # Get more detailed memory info |
| 295 | memory_stats = torch.cuda.memory_stats(device) |
| 296 | max_memory_allocated = memory_stats.get("allocated_bytes.all.peak", 0) / 1e9 |
| 297 | max_memory_reserved = memory_stats.get("reserved_bytes.all.peak", 0) / 1e9 |
| 298 | |
| 299 | # Get DDP info if available |
| 300 | ddp_info = "" |
| 301 | if dist.is_initialized(): |
| 302 | ddp_info = f" | DDP: rank={dist.get_rank()}, world_size={dist.get_world_size()}" |
| 303 | |
| 304 | logging.info( |
| 305 | f"Step {step} ({phase}): GPU memory - allocated: {memory_allocated:.2f}GB, reserved: {memory_reserved:.2f}GB, free: {memory_free:.2f}GB, peak_allocated: {max_memory_allocated:.2f}GB, peak_reserved: {max_memory_reserved:.2f}GB{ddp_info}" |
| 306 | ) |
| 307 | |
| 308 | |
| 309 | def train_loop(config: _config.TrainConfig): |
no outgoing calls
no test coverage detected