Function call_vllm_api_async

run_tessy.py:51–102

(session: aiohttp.ClientSession, api_url: str, model_name: str, prompt: str, max_tokens: int, temperature: float = 0.7, top_p: float = 0.8, top_k: int = 20, min_p: float = 0.0, max_retries: int = 3)

            return f"{user_message_formatted}<|start|>assistant<|channel|>analysis<|message|><|end|><|start|>assistant<|channel|>final<|message|>"

async def call_vllm_api_async(session: aiohttp.ClientSession, api_url: str, model_name: str, prompt: str, max_tokens: int, temperature: float = 0.7, top_p: float = 0.8, top_k: int = 20, min_p: float = 0.0, max_retries: int = 3):
# , stop_sequences: list = None):
    headers = {'Content-Type': 'application/json'}
    # print('model_name:', model_name)
    payload = {
        "model": model_name,
        "prompt": prompt, # prompt is complete now, including original prompt and generated content
        "max_tokens": max_tokens,
        "temperature": temperature,
        "top_p": top_p,
        "top_k": top_k,
        "min_p": min_p,
        "n": 1,
        "stream": False,
    }

    # if stop_sequences:
    #     payload["stop"] = stop_sequences

    for attempt in range(max_retries):
        try:
            async with session.post(api_url, headers=headers, json=payload, timeout=600) as response:
                response.raise_for_status()
                api_res = await response.json()
                generated_text = api_res['choices'][0]['text']

                # If return is empty, also treat as failure
                if not generated_text or len(generated_text.strip()) == 0:
                    print(f"API returned empty (attempt {attempt + 1}/{max_retries})")
                    if attempt < max_retries - 1:
                        await asyncio.sleep(2 ** attempt)  # Exponential backoff: 2s, 4s
                        continue
                    else:
                        return {"text": ""}

                return {"text": generated_text}
        except aiohttp.ClientError as e:
            print(f"API call failed (attempt {attempt + 1}/{max_retries}): {e}")
            if attempt < max_retries - 1:
                await asyncio.sleep(2 ** attempt)  # Exponential backoff: 2s, 4s
            else:
                print(f"Request URL: {api_url}, Payload: {payload}")
                return {"text": ""}
        except asyncio.TimeoutError:
            print(f"API call timeout (attempt {attempt + 1}/{max_retries})")
            if attempt < max_retries - 1:
                await asyncio.sleep(2 ** attempt)  # Exponential backoff: 2s, 4s
            else:
                print(f"Request URL: {api_url}, Payload: {payload}")
                return {"text": ""}

    return {"text": ""}

def classify_next_token_decision(batch_cur_texts, batch_generated_texts, current_model_name,
                                 classifier_tokenizer, classifier_model, max_length=512, classifier_batch_size=32):