(self, **config)
| 25 | |
| 26 | class Mind2Web(Task): |
| 27 | def __init__(self, **config): |
| 28 | self.count = config.pop("count", 100000) |
| 29 | self.range_min = config.pop("range_min", 0) |
| 30 | self.range_max = config.pop("range_max", 100) |
| 31 | cfg = Dict(config) |
| 32 | tokenizer = AutoTokenizer.from_pretrained(cfg.model.model_name_or_path) |
| 33 | # Load rank of candidates |
| 34 | candidate_results = None |
| 35 | if cfg.data.score_file is not None: |
| 36 | with open(cfg.data.score_file, "rb") as f: |
| 37 | candidate_results = pickle.load(f) |
| 38 | |
| 39 | self.test_dataset_dict = {} |
| 40 | for test_key, test_split_file in cfg.data.test_split_files.items(): |
| 41 | test_data = get_data_split( |
| 42 | cfg.data.data_path, |
| 43 | test_split_file, |
| 44 | candidate_results=candidate_results, |
| 45 | cache_dir=cfg.data.cache_path, |
| 46 | is_debug=bool(cfg.debug) |
| 47 | ) |
| 48 | self.test_dataset_dict[test_key] = MultiChoiceDataset( |
| 49 | test_data, |
| 50 | tokenizer=tokenizer, |
| 51 | neg_ratio=cfg.train.neg_ratio, |
| 52 | num_candidates=cfg.train.num_candidates, |
| 53 | max_context_len=cfg.train.max_context_len, |
| 54 | ) |
| 55 | # evaluation configs |
| 56 | random.seed(cfg.seed) |
| 57 | self.top_k = cfg.eval.topk |
| 58 | self.candidates_num = cfg.train.num_candidates |
| 59 | with open(cfg.llm_prompt, "r") as f: |
| 60 | self.prompt_template = json.load(f) |
| 61 | |
| 62 | super().__init__(**config) |
| 63 | |
| 64 | @property |
| 65 | def metrics(self) -> Dict: |
nothing calls this directly
no test coverage detected