()
| 116 | # IP_addr = '192.168.0.208' |
| 117 | Ip_addr = config['ServerConfig']['serverIp_'] |
| 118 | |
| 119 | r = create_redis_connection() |
| 120 | |
| 121 | |
| 122 | # 更新每个GPU的内存信息和利用率 |
| 123 | for i in range(dev_cnt): |
| 124 | redis_key = f"{Ip_addr}:{i}" |
| 125 | # now_gpuInfo_bytes = r.get(redis_key) |
| 126 | now_gpuInfo_bytes = r.hget(redis_key, 'gpu_info') |
| 127 | if now_gpuInfo_bytes: |
| 128 | now_gpuInfo = json.loads(now_gpuInfo_bytes.decode('utf-8')) |
| 129 | else: |
| 130 | now_gpuInfo = {} |
| 131 | #更新gpu信息 |
| 132 | handle = pynvml.nvmlDeviceGetHandleByIndex(i) |
| 133 | mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) |
| 134 | now_gpuInfo['memory_total'] = mem_info.total |
| 135 | now_gpuInfo['memory_free'] = mem_info.free |
| 136 | now_gpuInfo['memory_used'] = mem_info.used |
| 137 | # 获取GPU利用率 |
| 138 | util = pynvml.nvmlDeviceGetUtilizationRates(handle) |
| 139 | # gpu_util = util.gpu |
| 140 | now_gpuInfo['utilization'] = util.gpu |
| 141 | # now_gpuInfo['Job_num'] = i + 1 |
| 142 | gpuInfo.append(now_gpuInfo) |
| 143 | # r.set(redis_key, json.dumps(now_gpuInfo)) |
| 144 | r.hset(redis_key, 'gpu_info', json.dumps(now_gpuInfo)) |
| 145 | logging.debug(gpuInfo) |
| 146 | # 清理，释放NVML资源 |
| 147 | pynvml.nvmlShutdown() |
| 148 | |
| 149 | #client任务结束后，减少GPU的Job_num |
| 150 | def reduce_gpu_job(data): |
| 151 | # 解析data字符串，提取GPU ID |
| 152 | gpu_ids = data.split(':')[1].split(',') |
| 153 | config = load_runtime_config() |
| 154 | r = create_redis_connection() |
| 155 | |
| 156 | Ip_addr = config['ServerConfig']['serverIp_'] |
| 157 | for gpu_id in gpu_ids: |
| 158 | key = f"{Ip_addr}:{gpu_id}" |
| 159 | if r.exists(key): |
| 160 | gpu_info = json.loads(r.hget(key, 'gpu_info').decode('utf-8')) |
| 161 | gpu_info['Job_num'] -= 1 |
| 162 | r.hset(key, 'gpu_info', json.dumps(gpu_info)) |
| 163 | gpu_info_update = json.loads(r.hget(key, 'gpu_info').decode('utf-8')) |
| 164 | logging.debug("gpu_info:", gpu_info_update) |
no test coverage detected