Function call_evaluation_vllm

eval/src/eval_common.py:337–374

Evaluate a single image using a local vLLM server.

(task)

# ============================================================

def call_evaluation_vllm(task):
    """Evaluate a single image using a local vLLM server."""
    from vllm_request import evaluate_batch

    index = task["index"]
    prompt = task["prompt"]
    testpoint = task["testpoint"]
    test_desc = task["test_desc"]
    img_path = task["img_path"]
    lang = task["lang"]
    max_retries = task["max_retries"]
    api_url = task["api_url"]

    system_prompt = build_system_prompt(prompt, testpoint, test_desc, lang)
    payload = [{"images": [img_path], "problem": system_prompt}]

    text = None
    for attempt in range(max_retries + 1):
        result = evaluate_batch(payload, api_url)[0]

        if not result.get("success"):
            print(f"vLLM request failed (attempt {attempt + 1}/{max_retries + 1})")
            continue

        text = result["model_output"]
        parsed = parse_evaluation_response(text, testpoint)
        if parsed is not None:
            analysis, score = parsed
            result_json = {
                "prompt": prompt,
                "testpoint": testpoint,
                "analysis": analysis,
                "score": score,
            }
            return make_result(index, testpoint, prompt, img_path, text, result_json)

    # All retries exhausted
    return make_result(index, testpoint, prompt, img_path, text)


# ============================================================