()
| 523 | return final_results |
| 524 | |
| 525 | def main(): |
| 526 | parser = argparse.ArgumentParser(description="Evaluate text-only agent on LoComo dataset (Parallel)") |
| 527 | parser.add_argument("--dataset", type=str, default="data/locomo10.json", |
| 528 | help="Path to the dataset file") |
| 529 | parser.add_argument("--model", type=str, default="Qwen2.5-3B-Instruct", |
| 530 | help="Model to use") |
| 531 | parser.add_argument("--output", type=str, default=None, |
| 532 | help="Path to save evaluation results") |
| 533 | parser.add_argument("--ratio", type=float, default=1.0, |
| 534 | help="Ratio of dataset to evaluate (0.0 to 1.0)") |
| 535 | parser.add_argument("--backend", type=str, default="sglang", |
| 536 | help="Backend to use (openai, ollama, or sglang)") |
| 537 | parser.add_argument("--temperature_c5", type=float, default=0.5, |
| 538 | help="Temperature for category 5 questions") |
| 539 | parser.add_argument("--retrieve_k", type=int, default=10, |
| 540 | help="Number of memories to retrieve") |
| 541 | parser.add_argument("--sglang_host", type=str, default="http://localhost", |
| 542 | help="SGLang server host (for sglang backend)") |
| 543 | parser.add_argument("--sglang_port", type=int, default=30000, |
| 544 | help="SGLang server port (for sglang backend)") |
| 545 | parser.add_argument("--num_workers", type=int, default=10, |
| 546 | help="Number of parallel workers") |
| 547 | parser.add_argument("--out_dir", type=str, default=None, |
| 548 | help="Output directory to save results") |
| 549 | args = parser.parse_args() |
| 550 | |
| 551 | if args.ratio <= 0.0 or args.ratio > 1.0: |
| 552 | raise ValueError("Ratio must be between 0.0 and 1.0") |
| 553 | |
| 554 | # Convert relative path to absolute path |
| 555 | dataset_path = os.path.join(os.path.dirname(__file__), args.dataset) |
| 556 | if args.output: |
| 557 | output_path = os.path.join(os.path.dirname(__file__), args.output) |
| 558 | else: |
| 559 | output_path = None |
| 560 | |
| 561 | evaluate_dataset( |
| 562 | dataset_path, |
| 563 | args.model, |
| 564 | output_path, |
| 565 | args.ratio, |
| 566 | args.backend, |
| 567 | args.temperature_c5, |
| 568 | args.retrieve_k, |
| 569 | args.sglang_host, |
| 570 | args.sglang_port, |
| 571 | args.num_workers, |
| 572 | args.out_dir |
| 573 | ) |
| 574 | |
| 575 | if __name__ == "__main__": |
| 576 | main() |
no test coverage detected