(self, url,ports, gpus, generate_model,startup_delay=30)
| 144 | |
| 145 | class InstanceManager: |
| 146 | def __init__(self, url,ports, gpus, generate_model,startup_delay=30): |
| 147 | self.ports = ports |
| 148 | self.gpus = gpus |
| 149 | self.base_url=url |
| 150 | self.instances = [] |
| 151 | self.lock = threading.Lock() |
| 152 | self.current_instance = 0 # 用于轮询策略 |
| 153 | self.generate_model=generate_model |
| 154 | self.TOTAL_TOKEN_COST = 0 |
| 155 | self.TOTAL_API_CALL_COST = 0 |
| 156 | for port, gpu in zip(self.ports, self.gpus): |
| 157 | self.instances.append({"port": port, "load": 0}) |
| 158 | def reset_token_cost(self): |
| 159 | """重置总的token消耗和API调用次数""" |
| 160 | self.TOTAL_TOKEN_COST = 0 |
nothing calls this directly
no outgoing calls
no test coverage detected