(self)
| 94 | self._evt.set() |
| 95 | |
| 96 | def _trigger_epoch(self): |
| 97 | # Don't do this in after_epoch because |
| 98 | # before,after_epoch are supposed to be extremely fast by design. |
| 99 | if not self._enabled: |
| 100 | return |
| 101 | try: |
| 102 | stats = self._queue.get(timeout=60) |
| 103 | except queue.Empty: |
| 104 | if self._proc.is_alive(): |
| 105 | raise RuntimeError("GPUUtilization.worker() is stuck. This is a bug.") |
| 106 | else: |
| 107 | raise RuntimeError("GPUUtilization.worker() process is killed unexpectedly.") |
| 108 | |
| 109 | if isinstance(stats, int) and stats == -1: |
| 110 | from ..train.base import StopTraining |
| 111 | raise StopTraining("GPUUtilizationTracker.worker has failed.") |
| 112 | for idx, dev in enumerate(self._devices): |
| 113 | self.trainer.monitors.put_scalar('GPUUtil/{}'.format(dev), stats[idx]) |
| 114 | |
| 115 | def _after_train(self): |
| 116 | if self._enabled: |
nothing calls this directly
no test coverage detected