TaskManager indexes all tasks from the default `lm_eval/tasks/` and an optional directory if provided.
| 12 | |
| 13 | |
| 14 | class TaskManager: |
| 15 | """TaskManager indexes all tasks from the default `lm_eval/tasks/` |
| 16 | and an optional directory if provided. |
| 17 | |
| 18 | """ |
| 19 | def __init__( |
| 20 | self, |
| 21 | verbosity="INFO", |
| 22 | include_path=None |
| 23 | ) -> None: |
| 24 | |
| 25 | self.verbosity = verbosity |
| 26 | self.include_path = include_path |
| 27 | self.logger = utils.eval_logger |
| 28 | self.logger.setLevel(getattr(logging, f"{verbosity}")) |
| 29 | |
| 30 | self._task_index = self.initialize_tasks( |
| 31 | include_path=include_path |
| 32 | ) |
| 33 | self._all_tasks = sorted(list(self._task_index.keys())) |
| 34 | |
| 35 | self.task_group_map = collections.defaultdict(list) |
| 36 | |
| 37 | def initialize_tasks(self, include_path: str = None): |
| 38 | """Creates an dictionary of tasks index. |
| 39 | |
| 40 | :param include_path: str = None |
| 41 | An additional path to be searched for tasks |
| 42 | |
| 43 | :return |
| 44 | Dictionary of task names as key and task metadata |
| 45 | """ |
| 46 | all_paths = [os.path.dirname(os.path.abspath(__file__)) + "/"] |
| 47 | if include_path is not None: |
| 48 | if isinstance(include_path, str): |
| 49 | include_path = [include_path] |
| 50 | all_paths.extend(include_path) |
| 51 | |
| 52 | task_index = {} |
| 53 | for task_dir in all_paths: |
| 54 | tasks = self._get_task_and_group(task_dir) |
| 55 | task_index = {**tasks, **task_index} |
| 56 | |
| 57 | return task_index |
| 58 | |
| 59 | @property |
| 60 | def all_tasks(self): |
| 61 | return self._all_tasks |
| 62 | |
| 63 | @property |
| 64 | def task_index(self): |
| 65 | return self._task_index |
| 66 | |
| 67 | def match_tasks(self, task_list): |
| 68 | return utils.pattern_match( |
| 69 | task_list, self.all_tasks |
| 70 | ) |
| 71 |
no outgoing calls
no test coverage detected