Wraps the first layer on each rank to receive from the previous rank.
| 10 | |
| 11 | |
| 12 | class PipelineFirstLayer(nn.Module): |
| 13 | """Wraps the first layer on each rank to receive from the previous rank.""" |
| 14 | |
| 15 | def __init__(self, original_layer, rank, group): |
| 16 | super().__init__() |
| 17 | dict.__setitem__(self, "_original_layer", original_layer) |
| 18 | self.rank = rank |
| 19 | self.group = group |
| 20 | |
| 21 | @property |
| 22 | def original_layer(self): |
| 23 | return self["_original_layer"] |
| 24 | |
| 25 | def __getattr__(self, name): |
| 26 | try: |
| 27 | return super().__getattr__(name) |
| 28 | except AttributeError: |
| 29 | return getattr(self["_original_layer"], name) |
| 30 | |
| 31 | def __call__(self, x, *args, **kwargs): |
| 32 | if self.rank != 0: |
| 33 | mx.eval(x) |
| 34 | x = mx.distributed.recv_like(x, self.rank - 1, group=self.group) |
| 35 | mx.eval(x) |
| 36 | return self.original_layer(x, *args, **kwargs) |
| 37 | |
| 38 | |
| 39 | class PipelineLastLayer(nn.Module): |
no outgoing calls
no test coverage detected