(args: Union[argparse.Namespace, None] = None)
| 232 | |
| 233 | |
| 234 | def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: |
| 235 | if not args: |
| 236 | # we allow for args to be passed externally, else we parse them ourselves |
| 237 | args = parse_eval_args() |
| 238 | |
| 239 | eval_logger = utils.eval_logger |
| 240 | eval_logger.setLevel(getattr(logging, f"{args.verbosity}")) |
| 241 | eval_logger.info(f"Verbosity set to {args.verbosity}") |
| 242 | os.environ["TOKENIZERS_PARALLELISM"] = "false" |
| 243 | |
| 244 | if args.predict_only: |
| 245 | args.log_samples = True |
| 246 | if (args.log_samples or args.predict_only) and not args.output_path: |
| 247 | assert args.output_path, "Specify --output_path" |
| 248 | |
| 249 | initialize_tasks(args.verbosity) |
| 250 | task_manager = TaskManager(args.verbosity, include_path=args.include_path) |
| 251 | |
| 252 | if args.limit: |
| 253 | eval_logger.warning( |
| 254 | " --limit SHOULD ONLY BE USED FOR TESTING." |
| 255 | "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." |
| 256 | ) |
| 257 | if args.include_path is not None: |
| 258 | eval_logger.info(f"Including path: {args.include_path}") |
| 259 | include_path(args.include_path) |
| 260 | |
| 261 | if args.tasks is None: |
| 262 | eval_logger.error("Need to specify task to evaluate.") |
| 263 | sys.exit() |
| 264 | elif args.tasks == "list": |
| 265 | eval_logger.info( |
| 266 | "Available Tasks:\n - {}".format("\n - ".join(task_manager.all_tasks)) |
| 267 | ) |
| 268 | sys.exit() |
| 269 | else: |
| 270 | if os.path.isdir(args.tasks): |
| 271 | import glob |
| 272 | |
| 273 | task_names = [] |
| 274 | yaml_path = os.path.join(args.tasks, "*.yaml") |
| 275 | for yaml_file in glob.glob(yaml_path): |
| 276 | config = utils.load_yaml_config(yaml_file) |
| 277 | task_names.append(config) |
| 278 | else: |
| 279 | task_list = args.tasks.split(",") |
| 280 | task_names = task_manager.match_tasks(task_list) |
| 281 | for task in [task for task in task_list if task not in task_names]: |
| 282 | if os.path.isfile(task): |
| 283 | config = utils.load_yaml_config(task) |
| 284 | task_names.append(config) |
| 285 | task_missing = [ |
| 286 | task for task in task_list if task not in task_names and "*" not in task |
| 287 | ] # we don't want errors if a wildcard ("*") task name was used |
| 288 | |
| 289 | if task_missing: |
| 290 | missing = ", ".join(task_missing) |
| 291 | eval_logger.error( |
no test coverage detected