Function load_scenarios

src/evaluation/loader.py:45–75

Load scenarios from one or more files or directories. Supported inputs: 1. Existing JSON / JSONL scenario files. 2. A directory containing scenario subdirectories, each with ``groundtruth.txt``. For example: scenarios_data/ scenario_11/ groundtruth.txt scenario_1

(paths: Iterable[Path] | Path)

def load_scenarios(paths: Iterable[Path] | Path) -> list[Scenario]:
    """Load scenarios from one or more files or directories.

    Supported inputs:

    1. Existing JSON / JSONL scenario files.
    2. A directory containing scenario subdirectories, each with
       ``groundtruth.txt``. For example:

       scenarios_data/
  scenario_11/
    groundtruth.txt
  scenario_12/
    groundtruth.txt

    For folder-based scenarios, the folder name becomes the scenario id and
    ``groundtruth.txt`` becomes ``expected_answer``.
    """
    if isinstance(paths, (str, Path)):
        paths = [Path(paths)]

    out: list[Scenario] = []
    for p in paths:
        p = Path(p)

        if p.is_dir():
            out.extend(_load_scenario_dir(p))
        else:
            out.extend(_load_scenario_file(p))

    return out


def _load_scenario_dir(path: Path) -> list[Scenario]: