(self)
| 58 | return devices |
| 59 | |
| 60 | def _setup_graph(self): |
| 61 | # special heuristics for Horovod |
| 62 | from ..train import HorovodTrainer |
| 63 | if isinstance(self.trainer, HorovodTrainer): |
| 64 | if self.trainer.mpi_enabled(): |
| 65 | logger.warn("GPUUtilizationTracker is disabled under MPI.") |
| 66 | self._enabled = False |
| 67 | return |
| 68 | else: |
| 69 | self._devices = [self.trainer.hvd.local_rank()] |
| 70 | |
| 71 | if self._devices is None: |
| 72 | self._devices = self._guess_devices() |
| 73 | assert len(self._devices), "[GPUUtilizationTracker] No GPU device given!" |
| 74 | |
| 75 | self._evt = mp.Event() |
| 76 | self._stop_evt = mp.Event() |
| 77 | self._queue = mp.Queue() |
| 78 | self._proc = mp.Process(target=self.worker, args=( |
| 79 | self._evt, self._queue, self._stop_evt, self._devices)) |
| 80 | ensure_proc_terminate(self._proc) |
| 81 | start_proc_mask_signal(self._proc) |
| 82 | |
| 83 | def _before_train(self): |
| 84 | assert gpu_available_in_session(), "[GPUUtilizationTracker] needs GPU!" |
nothing calls this directly
no test coverage detected