Function main

utils/test_perplexity.py:539–604

()

def main():
    parser = argparse.ArgumentParser(description='Test model perplexity on multiple datasets')
    parser.add_argument('--model', '-m',
                        required=True,
                        help='Path to GGUF model file')
    parser.add_argument('--data-dir', '-d',
                        default='data',
                        help='Directory containing dataset folders (default: data)')
    parser.add_argument('--threads', '-t',
                        type=int,
                        default=16,
                        help='Number of threads (default: 16)')
    parser.add_argument('--ctx-size', '-c',
                        type=int,
                        default=512,
                        help='Context size (default: 512)')
    parser.add_argument('--output-dir', '-o',
                        default='perplexity_results',
                        help='Output directory for results (default: perplexity_results)')
    parser.add_argument('--llama-perplexity',
                        default='./build/bin/llama-perplexity',
                        help='Path to llama-perplexity binary (default: ./build/bin/llama-perplexity)')
    parser.add_argument('--quick', '-q',
                        action='store_true',
                        help='Quick test mode: test all datasets with first 4096 characters and reduced context size (128)')
    parser.add_argument('--test-embeddings', '-e',
                        action='store_true',
                        help='Test different embedding quantization types (f32, f16, q8_0, q6_k, q5_0, q4_0, q3_k, tq2_0)')
    parser.add_argument('--csv-output',
                        help='Custom path for CSV output file (e.g., results/my_ppl_results.csv)')
    parser.add_argument('--quantize-bin',
                        default='./build/bin/llama-quantize',
                        help='Path to llama-quantize binary (default: ./build/bin/llama-quantize)')
    
    args = parser.parse_args()
    
    try:
        tester = PerplexityTester(
            model_path=args.model,
            llama_perplexity_bin=args.llama_perplexity,
            data_dir=args.data_dir,
            output_dir=args.output_dir,
            quick_mode=args.quick,
            quantize_bin=args.quantize_bin,
            test_embeddings=args.test_embeddings,
            csv_output=args.csv_output
        )
        
        tester.run_all_tests(
            threads=args.threads,
            ctx_size=args.ctx_size
        )
        
    except FileNotFoundError as e:
        print(f"❌ Error: {e}")
        return 1
    except KeyboardInterrupt:
        print("\n\n⚠️  Test interrupted by user")