For FSPD2, disable gradient synchronization for all model parameters.
(self:Accelerator, model)
| 44 | |
| 45 | @contextmanager |
| 46 | def no_sync(self:Accelerator, model): |
| 47 | '''For FSPD2, disable gradient synchronization for all model parameters.''' |
| 48 | context = nullcontext |
| 49 | if self.use_distributed: |
| 50 | context = getattr(model, "no_sync", context) |
| 51 | if self.is_fsdp2 and os.environ.get("ENABLE_FSDP2_NOSYNC","False") == "True": |
| 52 | model.set_requires_gradient_sync(False) |
| 53 | yield |
| 54 | model.set_requires_gradient_sync(True) |
| 55 | return |
| 56 | |
| 57 | with context(): |
| 58 | yield |
| 59 | |
| 60 | |
| 61 |
nothing calls this directly
no outgoing calls
no test coverage detected