Function evaluate_text

eval_suite/text_utils.py:54–80

Evaluate transcript text using an LLM model with retry logic. Args: text_eval_model: The LLM model wrapper to use for evaluation. transcript: The transcript text to evaluate. retry_limit: Maximum number of retry attempts on failure. Returns: dict: The e

(text_eval_model: LiteLLMWrapper, transcript: str, retry_limit: int)

def evaluate_text(text_eval_model: LiteLLMWrapper, transcript: str, retry_limit: int) -> dict:
    """
    Evaluate transcript text using an LLM model with retry logic.

    Args:
        text_eval_model: The LLM model wrapper to use for evaluation.
        transcript: The transcript text to evaluate.
        retry_limit: Maximum number of retry attempts on failure.

    Returns:
        dict: The evaluation results as a JSON object.

    Raises:
        ValueError: If all retry attempts fail.
    """
    # prompt = _text_eval.format(transcript=transcript)
    prompt = _text_eval_new.format(transcript=transcript)
    for attempt in range(retry_limit):
        try:
            evaluation = text_eval_model(_prepare_text_inputs(prompt))
            evaluation_json = extract_json(evaluation)
            evaluation_json = convert_score_fields(evaluation_json)
            return evaluation_json
        except Exception as e:
            print(f"Attempt {attempt + 1} failed: {e.__class__.__name__}: {e}")
            if attempt + 1 == retry_limit:
                raise ValueError("Reached maximum retry limit. Evaluation failed.") from None