OpenAI-style chat completions serving
| 67 | |
| 68 | |
| 69 | class OpenAIServingChat: |
| 70 | """ |
| 71 | OpenAI-style chat completions serving |
| 72 | """ |
| 73 | |
| 74 | def __init__( |
| 75 | self, |
| 76 | engine_client, |
| 77 | models, |
| 78 | pid, |
| 79 | ips, |
| 80 | max_waiting_time, |
| 81 | chat_template, |
| 82 | enable_mm_output: Optional[bool] = False, |
| 83 | tokenizer_base_url: Optional[str] = None, |
| 84 | ): |
| 85 | self.engine_client = engine_client |
| 86 | self.models = models |
| 87 | self.pid = pid |
| 88 | self.max_waiting_time = max_waiting_time |
| 89 | self.chat_template = chat_template |
| 90 | self.enable_mm_output = enable_mm_output |
| 91 | self.tokenizer_base_url = tokenizer_base_url |
| 92 | if ips is not None: |
| 93 | if isinstance(ips, list): |
| 94 | self.master_ip = ips[0] |
| 95 | else: |
| 96 | self.master_ip = ips.split(",")[0] |
| 97 | self.is_master_ip = get_host_ip() == self.master_ip |
| 98 | else: |
| 99 | self.master_ip = "0.0.0.0" |
| 100 | self.is_master_ip = True |
| 101 | api_server_logger.info(f"master ip: {self.master_ip}") |
| 102 | |
| 103 | def _check_master(self): |
| 104 | return self.engine_client.is_master or self.is_master_ip |
| 105 | |
| 106 | async def create_chat_completion(self, request: ChatCompletionRequest): |
| 107 | """ |
| 108 | Create a new chat completion using the specified parameters. |
| 109 | """ |
| 110 | tracing.trace_set_thread_info("API Server") |
| 111 | if not self._check_master(): |
| 112 | err_msg = ( |
| 113 | f"Only master node can accept completion request, please send request to master node: {self.master_ip}" |
| 114 | ) |
| 115 | api_server_logger.error(err_msg) |
| 116 | return ErrorResponse(error=ErrorInfo(message=err_msg, type=ErrorType.INTERNAL_ERROR)) |
| 117 | |
| 118 | if self.models: |
| 119 | is_supported, request.model = self.models.is_supported_model(request.model) |
| 120 | if not is_supported: |
| 121 | err_msg = f"Unsupported model: [{request.model}], support [{', '.join([x.name for x in self.models.model_paths])}] or default" |
| 122 | api_server_logger.error(err_msg) |
| 123 | return ErrorResponse( |
| 124 | error=ErrorInfo(message=err_msg, type=ErrorType.INTERNAL_ERROR, code=ErrorCode.MODEL_NOT_SUPPORT) |
| 125 | ) |
| 126 |
no outgoing calls