(self)
| 3380 | return self.engine_timers_cache |
| 3381 | |
| 3382 | def _autotuning_exit(self): |
| 3383 | if self.global_rank == 0: |
| 3384 | msg = self.timers.get_mean([ |
| 3385 | FORWARD_GLOBAL_TIMER, |
| 3386 | BACKWARD_GLOBAL_TIMER, |
| 3387 | STEP_GLOBAL_TIMER, |
| 3388 | ], reset=False) |
| 3389 | titer = 0.0 |
| 3390 | titer += msg[FORWARD_GLOBAL_TIMER] if FORWARD_GLOBAL_TIMER in msg else 0 |
| 3391 | titer += msg[BACKWARD_GLOBAL_TIMER] if BACKWARD_GLOBAL_TIMER in msg else 0 |
| 3392 | titer += msg[STEP_GLOBAL_TIMER] if STEP_GLOBAL_TIMER in msg else 0 |
| 3393 | titer *= self.gradient_accumulation_steps() |
| 3394 | msg["latency"] = titer |
| 3395 | msg["FLOPS_per_gpu"] = self.flops * 1_000_000 * self.gradient_accumulation_steps() / titer |
| 3396 | msg["throughput"] = self.train_batch_size() * 1_000_000 / \ |
| 3397 | msg["latency"] |
| 3398 | print_json_dist(msg, [0], path=self.autotuning_metric_path()) |
| 3399 | log_dist( |
| 3400 | f"Wrote metrics to {self.autotuning_metric_path()}, {os.path.abspath(self.autotuning_metric_path())}", |
| 3401 | ranks=[0]) |
| 3402 | import atexit |
| 3403 | atexit.register(print, "Autotuning: done with running current ds config.") |
| 3404 | exit() |
| 3405 | |
| 3406 | def _write_monitor(self): |
| 3407 | if self.global_rank == 0: |
no test coverage detected