Query nvidia-smi for GPU stats.
()
| 41 | "name": parts[1], |
| 42 | "utilization": float(parts[2]), |
| 43 | "memUsed": round(float(parts[3]) / 1024, 2), # MiB -> GiB |
| 44 | "memTotal": round(float(parts[4]) / 1024, 2), |
| 45 | "temperature": int(float(parts[5])), |
| 46 | }) |
| 47 | return gpus |
| 48 | except (FileNotFoundError, subprocess.TimeoutExpired, Exception): |
| 49 | return [] |
| 50 | |
| 51 | |
| 52 | def get_npu_stats() -> list[dict]: |
| 53 | """Query npu-smi for Ascend NPU stats.""" |
| 54 | try: |
| 55 | result = subprocess.run(["npu-smi", "info"], capture_output=True, text=True, timeout=5) |
| 56 | if result.returncode != 0: |
| 57 | return [] |
| 58 | import re |
| 59 | npus, lines = [], result.stdout.strip().split("\n") |
| 60 | data_lines = [] |
| 61 | for line in lines: |
| 62 | line = line.strip() |
| 63 | if line.startswith("|") and "===" not in line and "---" not in line: |
| 64 | if "npu-smi" in line or "Version" in line: continue |
| 65 | if "NPU" in line and "Name" in line: continue |
| 66 | if "Chip" in line and "Bus-Id" in line: continue |
| 67 | if "Process" in line: break |
| 68 | data_lines.append(line) |
| 69 | for i in range(0, len(data_lines) - 1, 2): |
| 70 | cells1 = [c.strip() for c in data_lines[i].split("|") if c.strip()] |
| 71 | cells2 = [c.strip() for c in data_lines[i + 1].split("|") if c.strip()] |
| 72 | if len(cells1) < 3 or len(cells2) < 3: continue |
| 73 | id_name = cells1[0].split() |
| 74 | try: npu_id = int(id_name[0]) |
| 75 | except (ValueError, IndexError): continue |
| 76 | name = f"Ascend {id_name[1]}" if len(id_name) > 1 else "Ascend NPU" |
| 77 | stats1 = cells1[2].split() |
| 78 | temperature = int(stats1[1]) if len(stats1) >= 2 else 0 |
no test coverage detected