(self, params)
| 191 | self.remove_worker(worker_name) |
| 192 | |
| 193 | def worker_api_generate_stream(self, params): |
| 194 | worker_addr = self.get_worker_address(params["model"]) |
| 195 | if not worker_addr: |
| 196 | logger.info(f"no worker: {params['model']}") |
| 197 | ret = { |
| 198 | "text": server_error_msg, |
| 199 | "error_code": 2, |
| 200 | } |
| 201 | yield json.dumps(ret).encode() + b"\0" |
| 202 | |
| 203 | try: |
| 204 | response = requests.post(worker_addr + "/worker_generate_stream", |
| 205 | json=params, stream=True, timeout=5) |
| 206 | for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): |
| 207 | if chunk: |
| 208 | yield chunk + b"\0" |
| 209 | except requests.exceptions.RequestException as e: |
| 210 | logger.info(f"worker timeout: {worker_addr}") |
| 211 | ret = { |
| 212 | "text": server_error_msg, |
| 213 | "error_code": 3, |
| 214 | } |
| 215 | yield json.dumps(ret).encode() + b"\0" |
| 216 | |
| 217 | |
| 218 | # Let the controller act as a worker to achieve hierarchical |
no test coverage detected