Method generate_text_asy

tools/utils.py:223–267

发送请求到选择的实例

(self, prompt,system_prompt=None, history_messages=[], **kwargs)

        
        return response_message
    async def generate_text_asy(self, prompt,system_prompt=None, history_messages=[], **kwargs):
        """发送请求到选择的实例"""
        port = self.get_available_instance()
        base_url = f"{self.base_url}:{port}/v1"
        messages = []
        if system_prompt:
            messages.append({"role": "system", "content": system_prompt})

        # Get the cached response if having-------------------
       
        if len(history_messages)>1:
            history_messages[0]['content'] = truncate_text(history_messages[0]['content'], max_tokens=3000)
            history_messages[1]['content'] = truncate_text(history_messages[1]['content'], max_tokens=25000)
        messages.extend(history_messages)
        messages.append({"role": "user", "content": prompt})
        try:
            cur_token_cost = len(tokenizer.encode(messages[0]['content']))
            if cur_token_cost>31000:
                cur_token_cost = 31000
                messages[0]['content'] = truncate_text(messages[0]['content'], max_tokens=31000)
            
            # logging api call cost
            self.TOTAL_API_CALL_COST += 1
            response = requests.post(
            f"{base_url}/chat/completions",
            json={
                "model": self.generate_model,
                "messages": messages,
                **kwargs,
                "chat_template_kwargs": {"enable_thinking": False}
            },
            timeout=240
        )
            response.raise_for_status()
            res=json.loads(response.content)
            self.TOTAL_TOKEN_COST += res["usage"]["prompt_tokens"]
            response_message = res["choices"][0]["message"]['content']#对结果进行后处理
        except Exception as e:
            print(f"Retry for Error: {e}")
            response = ""    
            response_message=""
        
        
        
        return response_message