(self, global_step=False, report_speed=True)
| 196 | self.start_time = time.time() |
| 197 | |
| 198 | def stop(self, global_step=False, report_speed=True): |
| 199 | if not self.started: |
| 200 | return |
| 201 | self.started = False |
| 202 | self.micro_step_count += 1 |
| 203 | if global_step: |
| 204 | self.global_step_count += 1 |
| 205 | |
| 206 | if self.start_time > 0: |
| 207 | get_accelerator().synchronize() |
| 208 | self.end_time = time.time() |
| 209 | duration = self.end_time - self.start_time |
| 210 | self.total_elapsed_time += duration |
| 211 | self.step_elapsed_time += duration |
| 212 | |
| 213 | if global_step: |
| 214 | if report_speed and self.global_step_count % self.steps_per_output == 0: |
| 215 | self.logging( |
| 216 | "epoch={}/micro_step={}/global_step={}, RunningAvgSamplesPerSec={}, CurrSamplesPerSec={}, " |
| 217 | "MemAllocated={}GB, MaxMemAllocated={}GB".format( |
| 218 | self.epoch_count, |
| 219 | self.micro_step_count, |
| 220 | self.global_step_count, |
| 221 | self.avg_samples_per_sec(), |
| 222 | self.batch_size / self.step_elapsed_time, |
| 223 | round(get_accelerator().memory_allocated() / 1024**3, 2), |
| 224 | round(get_accelerator().max_memory_allocated() / 1024**3, 2), |
| 225 | )) |
| 226 | if self.monitor_memory: |
| 227 | virt_mem = psutil.virtual_memory() |
| 228 | swap = psutil.swap_memory() |
| 229 | self.logging("epoch={}/micro_step={}/global_step={}, vm %: {}, swap %: {}".format( |
| 230 | self.epoch_count, |
| 231 | self.micro_step_count, |
| 232 | self.global_step_count, |
| 233 | virt_mem.percent, |
| 234 | swap.percent, |
| 235 | )) |
| 236 | self.step_elapsed_time = 0 |
| 237 | |
| 238 | def avg_samples_per_sec(self): |
| 239 | if self.global_step_count > 0: |
no test coverage detected