Function get_ppl

utils/poster_eval_utils.py:552–617

Compute perplexity for arbitrarily long *text* using a sliding‑window approach. Parameters ---------- text : str The input string (any length). model_name : str, optional HF Hub id of the model to use, by default "meta-llama/Llama-2-7b-hf". stride : int, optional

(
    text: str,
    model_name: str = "meta-llama/Llama-2-7b-hf",
    stride: int = 512,
)

    return math.exp(-sum(valid) / len(valid))

def get_ppl(
    text: str,
    model_name: str = "meta-llama/Llama-2-7b-hf",
    stride: int = 512,
) -> float:
    """Compute perplexity for arbitrarily long *text* using a sliding‑window approach.

    Parameters
    ----------
    text : str
        The input string (any length).
    model_name : str, optional
        HF Hub id of the model to use, by default "meta-llama/Llama-2-7b-hf".
    stride : int, optional
        Overlap between successive windows. 512 tends to work well for most
        Transformer LMs with a 2 k context. Increase it for higher accuracy at
        the cost of more compute.

    Returns
    -------
    float
        Per‑token perplexity under the given model.
    """
    # Load tokenizer / model once per call (cache makes subsequent calls cheap)
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.float16,
        device_map="auto",  # place on GPU if available
    )
    model.eval()

    # Encode the whole string in one shot
    encodings = tokenizer(text, return_tensors="pt")
    input_ids = encodings.input_ids[0]

    # Model context length (e.g. 2048 for Llama‑2)
    max_len = model.config.max_position_embeddings

    # --- Short input: fits in a single window --------------------------------
    if input_ids.size(0) <= max_len:
        with torch.no_grad():
            out = model(input_ids.unsqueeze(0).to(model.device), labels=input_ids.unsqueeze(0).to(model.device))
        return torch.exp(out.loss).item()

    # --- Long input: sliding window with overlap -----------------------------
    nlls = []  # negative‑log‑likelihoods (already multiplied by #tokens scored)
    for i in range(0, input_ids.size(0), stride):
        begin_loc = max(i + stride - max_len, 0)
        end_loc = min(i + stride, input_ids.size(0))
        trg_len = end_loc - i  # tokens we actually score in this window

        ids_chunk = input_ids[begin_loc:end_loc]
        labels = ids_chunk.clone()
        labels[:-trg_len] = -100  # mask out purely‑context tokens

        with torch.no_grad():
            out = model(ids_chunk.unsqueeze(0).to(model.device), labels=labels.unsqueeze(0).to(model.device))