Method __init__

AgentBench.old/src/tasks/os_interaction/task.py:238–277

(self, **kwargs)

        return configs

    def __init__(self, **kwargs):
        # TODO: load config here
        self.match_problem: bool = kwargs.pop("match_problem", True)
        self.check_problem: bool = kwargs.pop("check_problem", True)
        self.round_limit: int = kwargs.pop("round_limit", 8)
        self.data_config = kwargs.pop("data_config", None)
        if not self.data_config:
            raise ValueError("data_config must be set")
        self.docker_config = kwargs.pop("docker_config", None)
        if not self.docker_config:
            raise ValueError("docker_config must be set")
        configs: List[dict[str, Any]] = []
        matches = []
        for item in self.data_config["files"]:
            path = item["problem_file"]
            for file in glob.glob(path):
                if file.endswith(".json") or file.endswith(".jsonl"):
                    matches.append({
                        "problem_file": file,
                        "script_dir": item["script_dir"]
                    })
        self.data_config["files"] = matches
        for item in self.data_config["files"]:
            problem_file, problem_dir = item["problem_file"], item["script_dir"]
            single_file_configs = self._load_configs(problem_file, problem_dir)
            single_file_check_configs = []
            single_file_match_configs = []
            for idx, config in enumerate(single_file_configs):
                if config.check:
                    single_file_check_configs.append({"file": problem_file, "config": config, "index": idx})
                elif config.match:
                    single_file_match_configs.append({"file": problem_file, "config": config, "index": idx})
            print("Load %s, %d problems, %d check problems, %d match problems." % (problem_file, len(single_file_configs), len(single_file_check_configs), len(single_file_match_configs)))
            if self.match_problem:
                configs.extend(single_file_match_configs)
            if self.check_problem:
                configs.extend(single_file_check_configs)
        self.problem_configs = configs

        super().__init__(**kwargs)

    def metric(self, prediction: List[Dict], target: List[None]):
        files = []