Function classify_categories

train/eval.py:188–235

Uses OpenAI API to classify whether each main category has deficiencies (booleans). Returns a list of category names predicted as True.

(model_output_text: str)

def classify_categories(model_output_text: str) -> List[str]:
    """
    Uses OpenAI API to classify whether each main category has deficiencies (booleans).
    Returns a list of category names predicted as True.
    """
    input_text = extract_answer_content(model_output_text)
    main_categories = list(DEFICIENCY_CATEGORIES.keys())

    # Load prompt from file
    prompt_template = load_classify_categories_prompt()
    prompt = prompt_template.format(
        categories=json.dumps(main_categories, indent=2),
        input_text=input_text
    )

    try:
        client = openai.OpenAI(
            api_key=os.getenv("OPENAI_API_KEY"),
            base_url=os.getenv("OPENAI_API_BASE_URL")
        )

        model_type = os.getenv("MODEL_TYPE", "gpt-4o-2024-08-06")

        completion = client.chat.completions.create(
            model=model_type,
            messages=[
                {"role": "system", "content": "You are an expert in slide design analysis. Your output must be a valid JSON object with booleans per category."},
                {"role": "user", "content": prompt}
            ],
            response_format={"type": "json_object"},
            temperature=0.0
        )

        response_text = completion.choices[0].message.content
        if not response_text:
            return []

        parsed = CategoryBooleanResponse.model_validate_json(response_text)
        bools = {
            "Composition & Layout": getattr(parsed, "composition_layout", False),
            "Typography": getattr(parsed, "typography", False),
            "Imagery & Visualizations": getattr(parsed, "imagery_visualizations", False),
        }
        return [name for name, flag in bools.items() if flag]

    except Exception as e:
        print(f"Error calling OpenAI API or parsing response: {e}")
        return []


class MultiTaskTester: