| 18 | |
| 19 | |
| 20 | class PerplexityTester: |
| 21 | def __init__(self, model_path, llama_perplexity_bin="../build/bin/llama-perplexity", |
| 22 | data_dir="../data", output_dir="perplexity_results", quick_mode=False, |
| 23 | quantize_bin="../build/bin/llama-quantize", test_embeddings=False, csv_output=None): |
| 24 | self.model_path = Path(model_path) |
| 25 | self.llama_perplexity_bin = Path(llama_perplexity_bin) |
| 26 | self.quantize_bin = Path(quantize_bin) |
| 27 | self.data_dir = Path(data_dir) |
| 28 | self.output_dir = Path(output_dir) |
| 29 | self.quick_mode = quick_mode |
| 30 | self.test_embeddings = test_embeddings |
| 31 | self.csv_output = Path(csv_output) if csv_output else None |
| 32 | self.results = [] |
| 33 | self.created_models = set() # Track newly created model files |
| 34 | self.temp_files = [] # Track temporary files for cleanup |
| 35 | |
| 36 | # Embedding types to test |
| 37 | self.embedding_types = [ |
| 38 | ('F32', 'f32'), |
| 39 | ('F16', 'f16'), |
| 40 | ('Q8_0', 'q8_0'), |
| 41 | ('Q6_K', 'q6_k'), |
| 42 | ('Q5_0', 'q5_0'), |
| 43 | ('Q4_0', 'q4_0'), |
| 44 | ('Q3_K', 'q3_k'), |
| 45 | ('TQ2_0', 'tq2_0'), |
| 46 | ] |
| 47 | |
| 48 | # Create output directory |
| 49 | self.output_dir.mkdir(parents=True, exist_ok=True) |
| 50 | |
| 51 | # Verify llama-perplexity binary exists |
| 52 | if not self.llama_perplexity_bin.exists(): |
| 53 | raise FileNotFoundError(f"llama-perplexity binary not found: {self.llama_perplexity_bin}") |
| 54 | |
| 55 | # Verify quantize binary exists if testing embeddings |
| 56 | if self.test_embeddings and not self.quantize_bin.exists(): |
| 57 | raise FileNotFoundError(f"llama-quantize binary not found: {self.quantize_bin}") |
| 58 | |
| 59 | # Verify model file exists |
| 60 | if not self.model_path.exists(): |
| 61 | raise FileNotFoundError(f"Model file not found: {self.model_path}") |
| 62 | |
| 63 | def find_datasets(self): |
| 64 | """Find all test.txt files in dataset directories.""" |
| 65 | datasets = [] |
| 66 | |
| 67 | if not self.data_dir.exists(): |
| 68 | print(f"❌ Data directory not found: {self.data_dir}") |
| 69 | return datasets |
| 70 | |
| 71 | print(f"\n🔍 Searching for datasets in {self.data_dir}...") |
| 72 | |
| 73 | # Look for test.txt files in subdirectories |
| 74 | for dataset_dir in sorted(self.data_dir.iterdir()): |
| 75 | if dataset_dir.is_dir(): |
| 76 | test_file = dataset_dir / "test.txt" |
| 77 | if test_file.exists(): |