Print the amount of GPU memory currently allocated for each GPU.
(rank: int = None, message: str = '')
| 104 | |
| 105 | |
| 106 | def print_gpu_memory(rank: int = None, message: str = ''): |
| 107 | """Print the amount of GPU memory currently allocated for each GPU.""" |
| 108 | if torch.cuda.is_available(): |
| 109 | device_count = torch.cuda.device_count() |
| 110 | for i in range(device_count): |
| 111 | device = torch.device(f'cuda:{i}') |
| 112 | allocated_bytes = torch.cuda.memory_allocated(device) |
| 113 | if allocated_bytes == 0: |
| 114 | continue |
| 115 | print('*' * 40) |
| 116 | print(f'[{message} rank {rank} ] GPU {i}: {allocated_bytes / 1024**2:.2f} MB') |
| 117 | print('*' * 40) |
| 118 | |
| 119 | |
| 120 | def get_block_class_from_model(model: torch.nn.Module, block_class_name: str) -> torch.nn.Module: |
nothing calls this directly
no outgoing calls
no test coverage detected