Function compute_vlm_ppl

utils/poster_eval_utils.py:94–137

(content)

    return blocks

def compute_vlm_ppl(content):
    VLLM_BASE_URL = "http://localhost:7000/v1"
    MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct"

    client = OpenAI(
        api_key="EMPTY",            # vLLM ignores auth
        base_url=VLLM_BASE_URL,
        timeout=Timeout(5000)
    )

    resp = client.chat.completions.create(
        model=MODEL_ID,
        messages=[{
            "role": "user",
            "content": content,
        }],
        temperature=0.0,
        max_tokens=1, 
        logprobs=0,
        extra_body={
            "prompt_logprobs": 1,
            "echo": True 
        }
    )

    lp_list = resp.to_dict()["prompt_logprobs"]   # list[dict]
    total_lp = 0.0
    n_text   = 0

    for token_entry in lp_list:
        if not token_entry:
            continue
        # find the sub-entry with rank==1 (the real token)
        token_info = next(v for v in token_entry.values() if v["rank"] == 1)
        tok, lp = token_info["decoded_token"], token_info["logprob"]

        # skip image sentinels / padding
        if re.fullmatch(r"<\|?image[^>]*\|?>", tok):
            continue

        total_lp += lp
        n_text   += 1

    return math.exp(-total_lp / n_text)

def compute_interleaved_ppl(paper_name, poster_method):
    base_dir = f'eval_poster_markdown/{paper_name}/{poster_method}'