Resets the GPU clocks.
(
logger,
device_id=0,
was_persistence_mode_on=False,
current_power_limit=None,
)
| 506 | |
| 507 | |
| 508 | def reset_clocks( |
| 509 | logger, |
| 510 | device_id=0, |
| 511 | was_persistence_mode_on=False, |
| 512 | current_power_limit=None, |
| 513 | ): |
| 514 | """ |
| 515 | Resets the GPU clocks. |
| 516 | """ |
| 517 | logger.info("Trying to reset the GPU clocks for device: %d" % device_id) |
| 518 | |
| 519 | gpu_available = torch.cuda.device_count() > 0 |
| 520 | |
| 521 | if not gpu_available: |
| 522 | logger.warning("No GPUs available to reset the clocks.") |
| 523 | |
| 524 | # 1. Reset the memory clock. |
| 525 | proc_ret = subprocess.run( |
| 526 | ["nvidia-smi", "-i=%d" % device_id, "--reset-memory-clocks"], |
| 527 | stdout=subprocess.PIPE, |
| 528 | ) |
| 529 | if proc_ret.returncode: |
| 530 | logger.warning("Unable to reset the memory clock back to normal.") |
| 531 | else: |
| 532 | logger.info("Reset the memory clock back to normal.") |
| 533 | |
| 534 | # 2. Reset GPU clock. |
| 535 | proc_ret = subprocess.run( |
| 536 | ["nvidia-smi", "-i=%d" % device_id, "--reset-gpu-clocks"], |
| 537 | stdout=subprocess.PIPE, |
| 538 | ) |
| 539 | if proc_ret.returncode: |
| 540 | logger.warning("Unable to reset the GPU clock back to normal.") |
| 541 | else: |
| 542 | logger.info("Reset the graphics clock back to normal.") |
| 543 | |
| 544 | # 3. Reset application clocks. |
| 545 | proc_ret = subprocess.run( |
| 546 | ["nvidia-smi", "-i=%d" % device_id, "--reset-applications-clocks"], |
| 547 | stdout=subprocess.PIPE, |
| 548 | ) |
| 549 | if proc_ret.returncode: |
| 550 | logger.warning("Unable to reset the application clocks back to normal.") |
| 551 | else: |
| 552 | logger.info("Reset the application clocks back to normal.") |
| 553 | |
| 554 | # 4. Reset the power limit. |
| 555 | if current_power_limit: |
| 556 | proc_ret = subprocess.run( |
| 557 | [ |
| 558 | "nvidia-smi", |
| 559 | "-i=%d" % device_id, |
| 560 | "--power-limit=%f" % current_power_limit, |
| 561 | ], |
| 562 | stdout=subprocess.PIPE, |
| 563 | ) |
| 564 | |
| 565 | if proc_ret.returncode: |