Function _load_scenario_dir

src/evaluation/loader.py:78–123

Load scenario folders from a directory containing groundtruth.txt files. Supports folder names like: scenario_34/ groundtruth.txt The scenario id becomes "34" so it can join with trajectory files such as: traces/trajectories/34.json

(path: Path)

def _load_scenario_dir(path: Path) -> list[Scenario]:
    """Load scenario folders from a directory containing groundtruth.txt files.

    Supports folder names like:
        scenario_34/
          groundtruth.txt

    The scenario id becomes "34" so it can join with trajectory files such as:
        traces/trajectories/34.json
    """
    scenarios: list[Scenario] = []

    for child in sorted(path.iterdir()):
        if not child.is_dir():
            continue

        groundtruth_path = child / "groundtruth.txt"
        if not groundtruth_path.exists():
            continue

        scenario_id = child.name
        if scenario_id.startswith("scenario_"):
            scenario_id = scenario_id.removeprefix("scenario_")

        expected_answer = groundtruth_path.read_text(encoding="utf-8").strip()

        question_path = child / "question.txt"
        text = (
            question_path.read_text(encoding="utf-8").strip()
            if question_path.exists()
            else ""
        )

        scenarios.append(
            Scenario.from_raw(
                {
                    "id": scenario_id,
                    "text": text,
                    "type": "structured",
                    "expected_answer": expected_answer,
                    "scoring_method": "static_json",
                }
            )
        )

    return scenarios


def _load_scenario_file(path: Path) -> list[Scenario]: