| 24 | |
| 25 | |
| 26 | class MemoryBuffer: |
| 27 | def __init__(self, numel, dtype): |
| 28 | self.numel = numel |
| 29 | self.dtype = dtype |
| 30 | self.data = torch.zeros( |
| 31 | self.numel, |
| 32 | dtype=self.dtype, |
| 33 | device=torch.cuda.current_device(), |
| 34 | requires_grad=False, |
| 35 | ) |
| 36 | |
| 37 | def zero(self): |
| 38 | """Reset the buffer to zero.""" |
| 39 | self.data.zero_() |
| 40 | |
| 41 | def get(self, shape, start_index): |
| 42 | """Return a tensor with the input `shape` as a view into the |
| 43 | 1-D data starting at `start_index`.""" |
| 44 | end_index = start_index + shape.numel() |
| 45 | assert end_index <= self.numel, "requested tensor is out of the buffer range." |
| 46 | buffer_tensor = self.data[start_index:end_index] |
| 47 | buffer_tensor = buffer_tensor.view(shape) |
| 48 | return buffer_tensor |
| 49 | |
| 50 | |
| 51 | class DistributedDataParallelBase(MegatronModule, ABC): |