Maximizes the GPU clocks. Useful to do it before any type of performance benchmarking. :param device_id: The GPU device ID whose clocks should be maximized.
(logger, device_id=0)
| 301 | |
| 302 | |
| 303 | def maximize_clocks(logger, device_id=0): |
| 304 | """ |
| 305 | Maximizes the GPU clocks. Useful to do it before any type of performance |
| 306 | benchmarking. |
| 307 | :param device_id: The GPU device ID whose clocks should be maximized. |
| 308 | """ |
| 309 | logger.info("Trying to maximize the GPU clocks for device: %d" % device_id) |
| 310 | |
| 311 | gpu_available = torch.cuda.device_count() > 0 |
| 312 | |
| 313 | was_persistence_mode_on = False |
| 314 | current_power_limit = None |
| 315 | |
| 316 | if not gpu_available: |
| 317 | logger.warning("No GPUs available to maximize the clocks.") |
| 318 | return False, was_persistence_mode_on, current_power_limit |
| 319 | |
| 320 | # 1. Enable persistence mode if not already done. |
| 321 | proc_ret = subprocess.run( |
| 322 | [ |
| 323 | "nvidia-smi", |
| 324 | "--query-gpu=persistence_mode", |
| 325 | "--format=csv,nounits,noheader", |
| 326 | ], |
| 327 | stdout=subprocess.PIPE, |
| 328 | ) |
| 329 | if proc_ret.returncode: |
| 330 | logger.warning("Unable to query persistence mode.") |
| 331 | was_persistence_mode_on = False |
| 332 | else: |
| 333 | was_persistence_mode_on = proc_ret.stdout.decode() == "Enabled" |
| 334 | |
| 335 | if not was_persistence_mode_on: |
| 336 | proc_ret = subprocess.run( |
| 337 | ["nvidia-smi", "--persistence-mode=1"], stdout=subprocess.PIPE |
| 338 | ) |
| 339 | if proc_ret.returncode: |
| 340 | logger.error("Unable to set persistence mode.") |
| 341 | return False, was_persistence_mode_on, current_power_limit |
| 342 | else: |
| 343 | logger.info("Turned on persistence mode.") |
| 344 | |
| 345 | # 2. Disable auto boost before locking clocks. |
| 346 | proc_ret = subprocess.run( |
| 347 | ["nvidia-smi", "--auto-boost-default=DISABLED"], stdout=subprocess.PIPE |
| 348 | ) |
| 349 | if proc_ret.returncode: |
| 350 | logger.warning("Unable to turn off auto boost mode.") |
| 351 | else: |
| 352 | logger.info("Turned off auto-boost mode.") |
| 353 | |
| 354 | # 3. Maximize the power limits. |
| 355 | # Get the current value first and save it. |
| 356 | proc_ret = subprocess.run( |
| 357 | [ |
| 358 | "nvidia-smi", |
| 359 | "--query-gpu=power.limit", |
| 360 | "--format=csv,nounits,noheader", |