| 96 | |
| 97 | |
| 98 | def eval_dataset(self): |
| 99 | eval_func = self.eval_func |
| 100 | |
| 101 | rag_dataset_path = os.path.join(self.dataset_dir,self.query_file) |
| 102 | with open(rag_dataset_path, "r") as f: |
| 103 | data = json.load(f) |
| 104 | data = data['examples'] |
| 105 | |
| 106 | if os.path.exists(self.output_file_path): |
| 107 | results = [] |
| 108 | with open(self.output_file_path, "r") as f: |
| 109 | for line in f: |
| 110 | results.append(json.loads(line.strip())) |
| 111 | uid_already = [item['uid'] for item in results] |
| 112 | data = [item for item in data if item['uid'] not in uid_already] |
| 113 | |
| 114 | if self.workers_num == 1: |
| 115 | for item in tqdm(data): |
| 116 | result = eval_func(item) |
| 117 | if result is None: |
| 118 | continue |
| 119 | with open(self.output_file_path, "a") as f: |
| 120 | json.dump(result, f,ensure_ascii=False) |
| 121 | f.write("\n") |
| 122 | else: |
| 123 | with ThreadPoolExecutor(max_workers=self.workers_num) as executor: |
| 124 | futures = [executor.submit(eval_func, item) for item in data] |
| 125 | results = [] |
| 126 | for future in tqdm(as_completed(futures), total=len(futures), desc="Processing"): |
| 127 | result = future.result() |
| 128 | results.append(result) |
| 129 | if len(results) >= 3: |
| 130 | with open(self.output_file_path, "a") as f: |
| 131 | for res in results: |
| 132 | if res is None: |
| 133 | continue |
| 134 | f.write(json.dumps(res,ensure_ascii=False) + "\n") |
| 135 | results = [] |
| 136 | if results: |
| 137 | with open(self.output_file_path, "a") as f: |
| 138 | for res in results: |
| 139 | if res is None: |
| 140 | continue |
| 141 | f.write(json.dumps(res,ensure_ascii=False) + "\n") |
| 142 | |
| 143 | return self.output_file_path |
| 144 | |
| 145 | def eval_overall(self): |
| 146 | data = [] |