Method run_all_tests

utils/test_perplexity.py:326–439

Run perplexity tests on all datasets.

(self, threads=16, ctx_size=512)

            print(f"ℹ️  Keeping existing file: {model_path.name}")
    
    def run_all_tests(self, threads=16, ctx_size=512):
        """Run perplexity tests on all datasets."""
        datasets = self.find_datasets()
        
        if not datasets:
            print(f"\n❌ No datasets found in {self.data_dir}")
            print(f"   Make sure each dataset directory has a test.txt file")
            return
        
        # Quick mode: test all datasets but only first 4096 chars with smaller context
        if self.quick_mode:
            ctx_size = min(ctx_size, 128)  # Use smaller context in quick mode
            print(f"\n⚡ QUICK TEST MODE ENABLED")
            print(f"   - Testing all datasets with first 4096 characters only")
            print(f"   - Using reduced context size: {ctx_size}")
        
        # Determine models to test
        if self.test_embeddings:
            print(f"\n{'='*80}")
            print(f"🧪 EMBEDDING QUANTIZATION TEST MODE")
            print(f"{'='*80}")
            print(f"📦 Base model: {self.model_path.name}")
            print(f"🔢 Embedding types to test: {len(self.embedding_types)}")
            print(f"📊 Datasets: {len(datasets)}")
            print(f"🧵 Threads: {threads}")
            print(f"📏 Context size: {ctx_size}")
            print(f"{'='*80}")
            
            total_start = time.time()
            
            # Test each embedding type
            for i, (embedding_type, output_suffix) in enumerate(self.embedding_types, 1):
                print(f"\n\n{'#'*80}")
                print(f"[{i}/{len(self.embedding_types)}] Testing embedding type: {output_suffix} ({embedding_type})")
                print(f"{'#'*80}")
                
                # Quantize model
                quantized_model = self.quantize_embedding(embedding_type, output_suffix)
                
                if quantized_model is None:
                    print(f"⚠️  Skipping tests for {output_suffix} due to quantization failure")
                    continue
                
                # Test on all datasets
                for j, dataset in enumerate(datasets, 1):
                    print(f"\n[{j}/{len(datasets)}] Testing {dataset['name']} with {output_suffix}...")
                    
                    # Use quick dataset if in quick mode
                    test_path = dataset['path']
                    if self.quick_mode:
                        test_path = self.create_quick_dataset(dataset['path'])
                    
                    result = self.run_perplexity_test(
                        f"{dataset['name']}_embed-{output_suffix}",
                        test_path,
                        threads,
                        ctx_size,
                        model_override=quantized_model