(self)
| 311 | |
| 312 | |
| 313 | def eval_dataset(self): |
| 314 | eval_func = self.eval_func |
| 315 | |
| 316 | rag_dataset_path = os.path.join(self.dataset_dir,self.query_file) |
| 317 | with open(rag_dataset_path, "r") as f: |
| 318 | data = json.load(f) |
| 319 | data = data['examples'] |
| 320 | |
| 321 | if os.path.exists(self.output_file_path): |
| 322 | results = [] |
| 323 | with open(self.output_file_path, "r") as f: |
| 324 | for line in f: |
| 325 | results.append(json.loads(line.strip())) |
| 326 | uid_already = [item['uid'] for item in results] |
| 327 | data = [item for item in data if item['uid'] not in uid_already] |
| 328 | |
| 329 | if self.workers_num == 1: |
| 330 | for item in tqdm(data): |
| 331 | result = eval_func(item) |
| 332 | if result is None: |
| 333 | continue |
| 334 | with open(self.output_file_path, "a") as f: |
| 335 | json.dump(result, f,ensure_ascii=False) |
| 336 | f.write("\n") |
| 337 | else: |
| 338 | with ThreadPoolExecutor(max_workers=self.workers_num) as executor: |
| 339 | futures = [executor.submit(eval_func, item) for item in data] |
| 340 | results = [] |
| 341 | for future in tqdm(as_completed(futures), total=len(futures), desc="Processing"): |
| 342 | result = future.result() |
| 343 | results.append(result) |
| 344 | if len(results) >= 3: |
| 345 | with open(self.output_file_path, "a") as f: |
| 346 | for res in results: |
| 347 | if res is None: |
| 348 | continue |
| 349 | f.write(json.dumps(res,ensure_ascii=False) + "\n") |
| 350 | results = [] |
| 351 | if results: |
| 352 | with open(self.output_file_path, "a") as f: |
| 353 | for res in results: |
| 354 | if res is None: |
| 355 | continue |
| 356 | f.write(json.dumps(res,ensure_ascii=False) + "\n") |
| 357 | |
| 358 | return self.output_file_path |
| 359 | |
| 360 | def arg_parse(): |
| 361 | import argparse |
no test coverage detected